From 8e2361ae797deb283328c67f38484f0836b30ad0 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 9 Nov 2020 10:08:07 -0800 Subject: [PATCH 01/37] tap.sh: Update after rebasing onto main --- Makefile | 17 ++++++++++---- examples/fluids/README.md | 23 ++++++++++++++++++- examples/fluids/{ => navier-stokes}/Makefile | 0 .../fluids/{ => navier-stokes}/advection.h | 0 .../fluids/{ => navier-stokes}/advection2d.h | 0 examples/fluids/{ => navier-stokes}/common.h | 0 .../{ => navier-stokes}/densitycurrent.h | 0 .../fluids/{ => navier-stokes}/navierstokes.c | 0 .../{ => navier-stokes}/stdoutParsing.py | 0 tests/junit.py | 6 +++-- tests/tap.sh | 8 ++++--- 11 files changed, 43 insertions(+), 11 deletions(-) rename examples/fluids/{ => navier-stokes}/Makefile (100%) rename examples/fluids/{ => navier-stokes}/advection.h (100%) rename examples/fluids/{ => navier-stokes}/advection2d.h (100%) rename examples/fluids/{ => navier-stokes}/common.h (100%) rename examples/fluids/{ => navier-stokes}/densitycurrent.h (100%) rename examples/fluids/{ => navier-stokes}/navierstokes.c (100%) rename examples/fluids/{ => navier-stokes}/stdoutParsing.py (100%) diff --git a/Makefile b/Makefile index 9a4d036589..f66e50545c 100644 --- a/Makefile +++ b/Makefile @@ -208,8 +208,9 @@ nekexamples := $(OBJDIR)/nek-bps petscexamples.c := $(wildcard examples/petsc/*.c) petscexamples := $(petscexamples.c:examples/petsc/%.c=$(OBJDIR)/petsc-%) # Fluid Dynamics Examples -fluidsexamples.c := $(sort $(wildcard examples/fluids/*.c)) -fluidsexamples := $(fluidsexamples.c:examples/fluids/%.c=$(OBJDIR)/fluids-%) +fluidsdir := navier-stokes shallow-water +fluidsexamples.c := $(sort $(foreach dir,$(fluidsdir), $(wildcard examples/fluids/$(dir)/*.c))) +fluidsexamples := $(foreach dir,$(fluidsdir), $(fluidsexamples.c:examples/fluids/$(dir)/%.c=$(OBJDIR)/fluids-%)) # Solid Mechanics Examples solidsexamples.c := $(sort $(wildcard examples/solids/*.c)) solidsexamples := $(solidsexamples.c:examples/solids/%.c=$(OBJDIR)/solids-%) @@ -525,10 +526,15 @@ $(OBJDIR)/petsc-% : examples/petsc/%.c $(libceed) $(ceed.pc) | $$(@D)/.DIR PETSC_DIR="$(abspath $(PETSC_DIR))" OPT="$(OPT)" $* mv examples/petsc/$* $@ -$(OBJDIR)/fluids-% : examples/fluids/%.c $(libceed) $(ceed.pc) | $$(@D)/.DIR - +$(MAKE) -C examples/fluids CEED_DIR=`pwd` \ +$(OBJDIR)/fluids-navierstokes : examples/fluids/navier-stokes/navierstokes.c $(libceed) $(ceed.pc) | $$(@D)/.DIR + +$(MAKE) -C examples/fluids/navier-stokes CEED_DIR=`pwd` \ PETSC_DIR="$(abspath $(PETSC_DIR))" OPT="$(OPT)" $* - mv examples/fluids/$* $@ + mv examples/fluids/navier-stokes/navierstokes $@ + +$(OBJDIR)/fluids-shallowwater : examples/fluids/shallow-water/shallowwater.c $(libceed) $(ceed.pc) | $$(@D)/.DIR + +$(MAKE) -C examples/fluids/shallow-water CEED_DIR=`pwd` \ + PETSC_DIR="$(abspath $(PETSC_DIR))" OPT="$(OPT)" $* + mv examples/fluids/shallow-water/shallowwater $@ $(OBJDIR)/solids-% : examples/solids/%.c $(libceed) $(ceed.pc) | $$(@D)/.DIR +$(MAKE) -C examples/solids CEED_DIR=`pwd` \ @@ -602,6 +608,7 @@ ceedexamples : $(examples) nekexamples : $(nekexamples) mfemexamples : $(mfemexamples) petscexamples : $(petscexamples) +fluidsexamples : $(fluidsexamples) # Benchmarks allbenchmarks = petsc-bps diff --git a/examples/fluids/README.md b/examples/fluids/README.md index d7f5c6a352..c8c23c18f3 100644 --- a/examples/fluids/README.md +++ b/examples/fluids/README.md @@ -1,6 +1,8 @@ +# libCEED: Fluid Dynamics Examples + ## libCEED: Navier-Stokes Example -This page provides a description of the Navier-Stokes example for the libCEED library, based on PETSc. +This section provides a description of the Navier-Stokes example for the libCEED library, based on PETSc. The Navier-Stokes problem solves the compressible Navier-Stokes equations in three dimensions using an explicit time integration. The state variables are mass density, momentum density, and energy density. @@ -267,3 +269,22 @@ or implicit formulation. The geometric factors and coordinate transformations required for the integration of the weak form are described in the file [`common.h`](common.h) + +## libCEED: Shallow-water Example + +This section provides a description of the shallow-water example for the libCEED library, based on PETSc. + +This solver computes the solution to the shallow-water equations on a cubed-sphere (i.e., +a tensor-product discrete sphere, obtained by projecting a cube inscribed in a sphere onto the surface +of the sphere). + +The main shallow-water solver for libCEED is defined in [`shallowwater.c`](shallowwater.c). + +Build by using + +`make` + +and run with + +`./shallowwater` + diff --git a/examples/fluids/Makefile b/examples/fluids/navier-stokes/Makefile similarity index 100% rename from examples/fluids/Makefile rename to examples/fluids/navier-stokes/Makefile diff --git a/examples/fluids/advection.h b/examples/fluids/navier-stokes/advection.h similarity index 100% rename from examples/fluids/advection.h rename to examples/fluids/navier-stokes/advection.h diff --git a/examples/fluids/advection2d.h b/examples/fluids/navier-stokes/advection2d.h similarity index 100% rename from examples/fluids/advection2d.h rename to examples/fluids/navier-stokes/advection2d.h diff --git a/examples/fluids/common.h b/examples/fluids/navier-stokes/common.h similarity index 100% rename from examples/fluids/common.h rename to examples/fluids/navier-stokes/common.h diff --git a/examples/fluids/densitycurrent.h b/examples/fluids/navier-stokes/densitycurrent.h similarity index 100% rename from examples/fluids/densitycurrent.h rename to examples/fluids/navier-stokes/densitycurrent.h diff --git a/examples/fluids/navierstokes.c b/examples/fluids/navier-stokes/navierstokes.c similarity index 100% rename from examples/fluids/navierstokes.c rename to examples/fluids/navier-stokes/navierstokes.c diff --git a/examples/fluids/stdoutParsing.py b/examples/fluids/navier-stokes/stdoutParsing.py similarity index 100% rename from examples/fluids/stdoutParsing.py rename to examples/fluids/navier-stokes/stdoutParsing.py diff --git a/tests/junit.py b/tests/junit.py index 157a4e60a4..b469d9c3b9 100755 --- a/tests/junit.py +++ b/tests/junit.py @@ -21,8 +21,10 @@ def get_source(test): return os.path.join('examples', 'mfem', test[5:] + '.cpp') elif test.startswith('nek-'): return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') - elif test.startswith('fluids-'): - return os.path.join('examples', 'fluids', test[7:] + '.c') + elif test.startswith('fluids-navierstokes'): + return os.path.join('examples', 'fluids', 'navier-stokes', test[7:] + '.c') + elif test.startswith('fluids-shallowwater'): + return os.path.join('examples', 'fluids', 'shallow-water', test[7:] + '.c') elif test.startswith('solids-'): return os.path.join('examples', 'solids', test[7:] + '.c') elif test.startswith('ex'): diff --git a/tests/tap.sh b/tests/tap.sh index d71755fb80..400f2442a8 100755 --- a/tests/tap.sh +++ b/tests/tap.sh @@ -31,12 +31,14 @@ elif [ ${1::4} == "nek-" ]; then for ((i=0;i<${numconfig};++i)); do allargs+=("$(awk -v i="$i" '/C_TESTARGS/,/\n/{j++}j==i+1{print; exit}' examples/nek/bps/${1:4}.usr* | cut -d\ -f2- )") done -elif [ ${1::7} == "fluids-" ]; then +elif [ ${1::7} == "fluids-navierstokes" ]; then # get all test configurations numconfig=$(grep -F //TESTARGS examples/fluids/${1:7}.c* | wc -l) for ((i=0;i<${numconfig};++i)); do - allargs+=("$(awk -v i="$i" '/\/\/TESTARGS/,/\n/{j++}j==i+1{print; exit}' examples/fluids/${1:7}.c | cut -d\ -f2- )") + allargs+=("$(awk -v i="$i" '/\/\/TESTARGS/,/\n/{j++}j==i+1{print; exit}' examples/fluids/navier-stokes/${1:7}.c | cut -d\ -f2- )") done +elif [ $1 == "fluids-shallowwater" ]; then + allargs=$(grep -F //TESTARGS examples/fluids/shallow-water/shallowwater.c* | cut -d\ -f2- ) elif [ ${1::7} == "solids-" ]; then allargs=$(grep -F //TESTARGS examples/solids/${1:7}.c* | cut -d\ -f2- ) elif [ ${1::2} == "ex" ]; then @@ -84,7 +86,7 @@ for ((i=0;i<${#backends[@]};++i)); do continue; fi - # Navier-Stokes test problem too large for most CUDA backends + # Navier-Stokes and shallow-water test problems too large for most CUDA backends if [[ "$backend" = *gpu* && "$backend" != /gpu/cuda/gen && \ ( "$1" = fluids-* ) ]]; then printf "ok $i0 # SKIP - test problem too large for $backend\n" From 5772553ecfb5351b8a9ce6ee130fba787ecc95a4 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Wed, 29 Apr 2020 14:58:06 -0600 Subject: [PATCH 02/37] Initial development of SWE solver --- examples/fluids/.gitignore | 15 +- examples/fluids/shallow-water/Makefile | 91 ++++ .../fluids/shallow-water/qfunctions/mass.h | 62 +++ .../shallow-water/qfunctions/setup_geo.h | 180 +++++++ .../shallow-water/qfunctions/shallowwater.h | 350 ++++++++++++++ examples/fluids/shallow-water/shallowwater.c | 421 ++++++++++++++++ examples/fluids/shallow-water/src/setup.c | 455 ++++++++++++++++++ examples/fluids/shallow-water/src/wrappers.c | 381 +++++++++++++++ examples/fluids/shallow-water/sw_headers.h | 180 +++++++ 9 files changed, 2130 insertions(+), 5 deletions(-) create mode 100644 examples/fluids/shallow-water/Makefile create mode 100644 examples/fluids/shallow-water/qfunctions/mass.h create mode 100644 examples/fluids/shallow-water/qfunctions/setup_geo.h create mode 100644 examples/fluids/shallow-water/qfunctions/shallowwater.h create mode 100644 examples/fluids/shallow-water/shallowwater.c create mode 100644 examples/fluids/shallow-water/src/setup.c create mode 100644 examples/fluids/shallow-water/src/wrappers.c create mode 100644 examples/fluids/shallow-water/sw_headers.h diff --git a/examples/fluids/.gitignore b/examples/fluids/.gitignore index 9a9519ca60..9dcf2cb216 100644 --- a/examples/fluids/.gitignore +++ b/examples/fluids/.gitignore @@ -1,5 +1,10 @@ -navierstokes -*.vtr -*.vts -*.bin* -*.vtu +navier-stokes/navierstokes +shallow-water/shallowwater +navier-stokes/*.vtr +navier-stokes/*.vts +navier-stokes/*.bin* +navier-stokes/*.vtu +shallow-water/*.vtr +shallow-water/*.vts +shallow-water/*.bin* +shallow-water/*.vtu diff --git a/examples/fluids/shallow-water/Makefile b/examples/fluids/shallow-water/Makefile new file mode 100644 index 0000000000..d9e2b9d419 --- /dev/null +++ b/examples/fluids/shallow-water/Makefile @@ -0,0 +1,91 @@ +# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +# reserved. See files LICENSE and NOTICE for details. +# +# This file is part of CEED, a collection of benchmarks, miniapps, software +# libraries and APIs for efficient high-order finite element and spectral +# element discretizations for exascale applications. For more information and +# source code availability see http://github.com/ceed. +# +# The CEED research is supported by the Exascale Computing Project (17-SC-20-SC) +# a collaborative effort of two U.S. Department of Energy organizations (Office +# of Science and the National Nuclear Security Administration) responsible for +# the planning and preparation of a capable exascale ecosystem, including +# software, applications, hardware, advanced system engineering and early +# testbed platforms, in support of the nation's exascale computing imperative. + +PETSc.pc := $(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig/PETSc.pc +CEED_DIR ?= ../.. +ceed.pc := $(CEED_DIR)/lib/pkgconfig/ceed.pc + +CC = $(call pkgconf, --variable=ccompiler $(PETSc.pc) $(ceed.pc)) +CFLAGS = -std=c99 \ + $(call pkgconf, --variable=cflags_extra $(PETSc.pc)) \ + $(call pkgconf, --cflags-only-other $(PETSc.pc)) \ + $(OPT) +CPPFLAGS = $(call pkgconf, --cflags-only-I $(PETSc.pc) $(ceed.pc)) \ + $(call pkgconf, --variable=cflags_dep $(PETSc.pc)) +LDFLAGS = $(call pkgconf, --libs-only-L --libs-only-other $(PETSc.pc) $(ceed.pc)) +LDFLAGS += $(patsubst -L%, $(call pkgconf, --variable=ldflag_rpath $(PETSc.pc))%, $(call pkgconf, --libs-only-L $(PETSc.pc) $(ceed.pc))) +LDLIBS = $(call pkgconf, --libs-only-l $(PETSc.pc) $(ceed.pc)) -lm + +OBJDIR := build +SRCDIR := src + +src.c := shallowwater.c $(sort $(wildcard $(SRCDIR)/*.c)) +src.o = $(src.c:%.c=$(OBJDIR)/%.o) + +all: shallowwater + +shallowwater: $(src.o) | $(PETSc.pc) $(ceed.pc) + $(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@ + +.SECONDEXPANSION: # to expand $$(@D)/.DIR +%/.DIR : + @mkdir -p $(@D) + @touch $@ + +# Output using the 216-color rules mode +rule_file = $(notdir $(1)) +rule_path = $(patsubst %/,%,$(dir $(1))) +last_path = $(notdir $(patsubst %/,%,$(dir $(1)))) +ansicolor = $(shell echo $(call last_path,$(1)) | cksum | cut -b1-2 | xargs -IS expr 2 \* S + 17) +emacs_out = @printf " %10s %s/%s\n" $(1) $(call rule_path,$(2)) $(call rule_file,$(2)) +color_out = @if [ -t 1 ]; then \ + printf " %10s \033[38;5;%d;1m%s\033[m/%s\n" \ + $(1) $(call ansicolor,$(2)) \ + $(call rule_path,$(2)) $(call rule_file,$(2)); else \ + printf " %10s %s\n" $(1) $(2); fi +# if TERM=dumb, use it, otherwise switch to the term one +output = $(if $(TERM:dumb=),$(call color_out,$1,$2),$(call emacs_out,$1,$2)) + +# if V is set to non-nil, turn the verbose mode +quiet = $(if $(V),$($(1)),$(call output,$1,$@);$($(1))) + +$(OBJDIR)/%.o : %.c | $$(@D)/.DIR + $(call quiet,CC) $(CPPFLAGS) $(CFLAGS) -c -o $@ $(abspath $<) + +# Rules for building the examples +#%: %.c + +print: $(PETSc.pc) $(ceed.pc) + $(info CC : $(CC)) + $(info CFLAGS : $(CFLAGS)) + $(info CPPFLAGS: $(CPPFLAGS)) + $(info LDFLAGS : $(LDFLAGS)) + $(info LDLIBS : $(LDLIBS)) + $(info OPT : $(OPT)) + @true + +clean: + $(RM) r $(OBJDIR) shallowwater *.vtu + +$(PETSc.pc): + $(if $(wildcard $@),,$(error \ + PETSc config not found. Please set PETSC_DIR and PETSC_ARCH)) + +.PHONY: all print clean + +pkgconf = $(shell pkg-config $1 | sed -e 's/^"//g' -e 's/"$$//g') + +-include $(src.o:%.o=%.d) diff --git a/examples/fluids/shallow-water/qfunctions/mass.h b/examples/fluids/shallow-water/qfunctions/mass.h new file mode 100644 index 0000000000..677b987f95 --- /dev/null +++ b/examples/fluids/shallow-water/qfunctions/mass.h @@ -0,0 +1,62 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Geometric factors and mass operator for shallow-water example using PETSc + +#ifndef mass_h +#define mass_h + +#ifndef __CUDACC__ +# include +#endif + +// ***************************************************************************** +// This QFunction applies the mass matrix to five interlaced fields. +// +// Inputs: +// u - Input vector at quadrature points +// qdata - Quadrature weights +// +// Output: +// v - Output vector at quadrature points +// +// ***************************************************************************** +CEED_QFUNCTION(Mass)(void *ctx, CeedInt Q, + const CeedScalar *const *in, CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], + (*qdata) = in[1]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + // *INDENT-ON* + + CeedPragmaSIMD + for (CeedInt i=0; i +#endif + +// ***************************************************************************** +// This QFunction sets up the geometric factors required for integration and +// coordinate transformations +// +// Reference (parent) 2D coordinates: X \in [-1, 1]^2 +// +// Global 3D physical coordinates given by the mesh: xx \in [-R, R]^3 +// with R radius of the sphere +// +// Local 3D physical coordinates on the 2D manifold: x \in [-l, l]^3 +// with l half edge of the cube inscribed in the sphere +// +// Change of coordinates matrix computed by the library: +// (physical 3D coords relative to reference 2D coords) +// dxx_j/dX_i (indicial notation) [3 * 2] +// +// Change of coordinates x (on the 2D manifold) relative to xx (phyisical 3D): +// dx_i/dxx_j (indicial notation) [3 * 3] +// +// Change of coordinates x (on the 2D manifold) relative to X (reference 2D): +// (by chain rule) +// dx_i/dX_j [3 * 2] = dx_i/dxx_k [3 * 3] * dxx_k/dX_j [3 * 2] +// +// modJ is given by the magnitude of the cross product of the columns of dx_i/dX_j +// +// The quadrature data is stored in the array qdata. +// +// We require the determinant of the Jacobian to properly compute integrals of +// the form: int( u v ) +// +// Qdata: modJ * w +// +// ***************************************************************************** +CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, + const CeedScalar *const *in, CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*X)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], + (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*w) = in[2]; + // Outputs + CeedScalar (*qdata)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + // *INDENT-ON* + + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; i +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +#ifndef physics_context_struct +#define physics_context_struct +typedef struct { + CeedScalar u0; + CeedScalar v0; + CeedScalar h0; + CeedScalar f; + CeedScalar g; + CeedScalar time; +} PhysicsContext_s; +typedef PhysicsContext_s *PhysicsContext; +#endif // physics_context_struct + +// ***************************************************************************** +// This QFunction sets the the initial conditions and boundary conditions +// +// For now we have sinusoidal terrain and constant reference height H0 +// +// ***************************************************************************** +static inline int Exact_SW(CeedInt dim, CeedScalar time, const CeedScalar X[], + CeedInt Nf, CeedScalar q[], void *ctx) { + + // Context + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar u0 = context->u0; + const CeedScalar v0 = context->v0; + const CeedScalar h0 = context->h0; + + // Setup + // -- Coordinates + const CeedScalar x = X[0]; + const CeedScalar y = X[1]; + + // Initial Conditions + q[0] = u0; + q[1] = v0; + q[2] = h0; + // Terrain topography, h_s + q[3] = sin(x) + cos(y); // put 0 for constant flat topography + // Reference height, H_0 + q[4] = 0; // flat + + // Return + return 0; +} + +// ***************************************************************************** +// Initial conditions for shallow-water +// ***************************************************************************** +CEED_QFUNCTION(ICsSW)(void *ctx, CeedInt Q, + const CeedScalar *const *in, CeedScalar *const *out) { + // Inputs + const CeedScalar (*X)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0]; + + // Outputs + CeedScalar (*q0)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; i] [CEED_DIR=] +// +// Sample runs: +// +// shallow-water +// shallow-water -ceed /cpu/self +// shallow-water -ceed /gpu/occa +// shallow-water -ceed /cpu/occa +// shallow-water -ceed /omp/occa +// shallow-water -ceed /ocl/occa +// + +/// @file +/// Shallow-water equations example using PETSc + +const char help[] = "Solve the shallow-water equations using PETSc and libCEED\n"; + +#include +#include +#include +#include +#include +#include "sw_headers.h" + +int main(int argc, char **argv) { + // PETSc context + PetscInt ierr; + MPI_Comm comm; + DM dm, dmviz; + TS ts; + TSAdapt adapt; + Mat J, interpviz; + User user; + Units units; + PetscInt degree, qextra, outputfreq, steps, contsteps; + PetscMPIInt rank; + PetscScalar ftime; + Vec Q, Qloc, Xloc; + const CeedInt ncompx = 3; + PetscInt viz_refine = 0; + PetscBool read_mesh, simplex, test; + PetscInt topodim = 2, ncompq = 5, lnodes; + // libCEED context + char ceedresource[PETSC_MAX_PATH_LEN] = "/cpu/self", + filename[PETSC_MAX_PATH_LEN]; + Ceed ceed; + CeedData ceeddata; + PetscScalar meter = 1e-2; // 1 meter in scaled length units + PetscScalar second = 1e-2; // 1 second in scaled time units + PetscScalar f = 0.0001; // mid-latitude Coriolis parameter + PetscScalar g = 9.81; // m/s^2 + PetscScalar mpersquareds; + // Performance context + double start, cpu_time_used; + + // Initialize PETSc + ierr = PetscInitialize(&argc, &argv, NULL, help); + if (ierr) return ierr; + + // Allocate PETSc context + ierr = PetscMalloc1(1, &user); CHKERRQ(ierr); + ierr = PetscMalloc1(1, &units); CHKERRQ(ierr); + + // Set up problem type command line option + //PetscFunctionListAdd(&icsflist, "sphere", &ICsSW); + //PetscFunctionListAdd(&qflist, "shallow-water", &SW); + + // Parse command line options + comm = PETSC_COMM_WORLD; + ierr = PetscOptionsBegin(comm, NULL, "Shallow-water equations in PETSc with libCEED", + NULL); CHKERRQ(ierr); + ierr = PetscOptionsString("-ceed", "CEED resource specifier", + NULL, ceedresource, ceedresource, + sizeof(ceedresource), NULL); CHKERRQ(ierr); + ierr = PetscOptionsBool("-test", "Run in test mode", + NULL, test=PETSC_FALSE, &test, NULL); CHKERRQ(ierr); + ierr = PetscOptionsInt("-viz_refine", + "Regular refinement levels for visualization", + NULL, viz_refine, &viz_refine, NULL); + CHKERRQ(ierr); + ierr = PetscOptionsScalar("-units_meter", "1 meter in scaled length units", + NULL, meter, &meter, NULL); CHKERRQ(ierr); + meter = fabs(meter); + ierr = PetscOptionsScalar("-units_second","1 second in scaled time units", + NULL, second, &second, NULL); CHKERRQ(ierr); + second = fabs(second); + outputfreq = 10; + ierr = PetscOptionsInt("-output_freq", "Frequency of output, in number of steps", + NULL, outputfreq, &outputfreq, NULL); CHKERRQ(ierr); + contsteps = 0; + ierr = PetscOptionsInt("-continue", "Continue from previous solution", + NULL, contsteps, &contsteps, NULL); CHKERRQ(ierr); + degree = 3; + ierr = PetscOptionsInt("-degree", "Polynomial degree of tensor product basis", + NULL, degree, °ree, NULL); CHKERRQ(ierr); + qextra = 2; + ierr = PetscOptionsInt("-qextra", "Number of extra quadrature points", + NULL, qextra, &qextra, NULL); CHKERRQ(ierr); + ierr = PetscOptionsScalar("-g", "Gravitational acceleration", + NULL, g, &g, NULL); CHKERRQ(ierr); + ierr = PetscOptionsScalar("-f", "Mid-latitude Coriolis parameter", + NULL, f, &f, NULL); CHKERRQ(ierr); + PetscStrncpy(user->outputfolder, ".", 2); + ierr = PetscOptionsString("-of", "Output folder", + NULL, user->outputfolder, user->outputfolder, + sizeof(user->outputfolder), NULL); CHKERRQ(ierr); + read_mesh = PETSC_FALSE; + ierr = PetscOptionsString("-mesh", "Read mesh from file", NULL, + filename, filename, sizeof(filename), &read_mesh); + CHKERRQ(ierr); + simplex = PETSC_FALSE; + ierr = PetscOptionsBool("-simplex", "Use simplices, or tensor product cells", + NULL, simplex, &simplex, NULL); CHKERRQ(ierr); + ierr = PetscOptionsEnd(); CHKERRQ(ierr); + + // Define derived units + mpersquareds = meter / PetscSqr(second); + + // Scale variables to desired units + f /= second; + g *= mpersquareds; + + // Set up the libCEED context + PhysicsContext_s ctxSetup = { + .u0 = 0., + .v0 = 0., + .h0 = .1, + .f = f, + .g = g, + .time = 0. + }; + + // Setup DM + if (read_mesh) { + ierr = DMPlexCreateFromFile(PETSC_COMM_WORLD, filename, PETSC_TRUE, &dm); + CHKERRQ(ierr); + } else { + // Create the mesh as a 0-refined sphere. This will create a cubic surface, not a box. + PetscBool simplex = PETSC_FALSE; + ierr = DMPlexCreateSphereMesh(PETSC_COMM_WORLD, topodim, simplex, &dm); + CHKERRQ(ierr); + // Set the object name + ierr = PetscObjectSetName((PetscObject)dm, "Sphere"); CHKERRQ(ierr); + // Distribute mesh over processes + { + DM dmDist = NULL; + PetscPartitioner part; + + ierr = DMPlexGetPartitioner(dm, &part); CHKERRQ(ierr); + ierr = PetscPartitionerSetFromOptions(part); CHKERRQ(ierr); + ierr = DMPlexDistribute(dm, 0, NULL, &dmDist); CHKERRQ(ierr); + if (dmDist) { + ierr = DMDestroy(&dm); CHKERRQ(ierr); + dm = dmDist; + } + } + // Refine DMPlex with uniform refinement using runtime option -dm_refine + ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE); CHKERRQ(ierr); + ierr = DMSetFromOptions(dm); CHKERRQ(ierr); + ierr = ProjectToUnitSphere(dm); CHKERRQ(ierr); + // View DMPlex via runtime option + ierr = DMViewFromOptions(dm, NULL, "-dm_view"); CHKERRQ(ierr); + } + + // Create DM + ierr = SetupDM(dm, degree, ncompq, topodim); + CHKERRQ(ierr); + + dmviz = NULL; + interpviz = NULL; + if (viz_refine) { + DM dmhierarchy[viz_refine+1]; + + ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE); CHKERRQ(ierr); + dmhierarchy[0] = dm; + for (PetscInt i = 0, d = degree; i < viz_refine; i++) { + Mat interp_next; + + ierr = DMRefine(dmhierarchy[i], MPI_COMM_NULL, &dmhierarchy[i+1]); + CHKERRQ(ierr); + ierr = DMSetCoarseDM(dmhierarchy[i+1], dmhierarchy[i]); CHKERRQ(ierr); + d = (d + 1) / 2; + if (i + 1 == viz_refine) d = 1; + ierr = SetupDM(dmhierarchy[i+1], degree, ncompq, topodim); CHKERRQ(ierr); + ierr = DMCreateInterpolation(dmhierarchy[i], dmhierarchy[i+1], + &interp_next, NULL); CHKERRQ(ierr); + if (!i) interpviz = interp_next; + else { + Mat C; + ierr = MatMatMult(interp_next, interpviz, MAT_INITIAL_MATRIX, + PETSC_DECIDE, &C); CHKERRQ(ierr); + ierr = MatDestroy(&interp_next); CHKERRQ(ierr); + ierr = MatDestroy(&interpviz); CHKERRQ(ierr); + interpviz = C; + } + } + for (PetscInt i=1; iM); CHKERRQ(ierr); + + // Initialize CEED + CeedInit(ceedresource, &ceed); + + // Setup libCEED's objects + ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); + ierr = SetupLibceed(dm, ceed, degree, topodim, qextra, + ncompx, ncompq, user, ceeddata, &ctxSetup); CHKERRQ(ierr); + + // Set up PETSc context + // Set up units structure + units->meter = meter; + units->second = second; + units->mpersquareds = mpersquareds; + + // Set up user structure + user->comm = comm; + user->outputfreq = outputfreq; + user->contsteps = contsteps; + user->units = units; + user->dm = dm; + user->dmviz = dmviz; + user->interpviz = interpviz; + user->ceed = ceed; + + // Calculate qdata and ICs + // Set up state global and local vectors + ierr = VecZeroEntries(Q); CHKERRQ(ierr); + ierr = VectorPlacePetscVec(user->q0ceed, Qloc); CHKERRQ(ierr); + + // Apply Setup Ceed Operators + ierr = DMGetCoordinatesLocal(dm, &Xloc); CHKERRQ(ierr); + ierr = VectorPlacePetscVec(ceeddata->xcorners, Xloc); CHKERRQ(ierr); + CeedOperatorApply(ceeddata->op_setup, ceeddata->xcorners, ceeddata->qdata, + CEED_REQUEST_IMMEDIATE); + ierr = ComputeLumpedMassMatrix(ceed, dm, ceeddata->Erestrictq, + ceeddata->basisq, ceeddata->Erestrictqdi, + ceeddata->qdata, + user->M); CHKERRQ(ierr); + + // Apply IC operator and fix multiplicity of initial state vector + ierr = ICs_FixMultiplicity(ceeddata->op_ics, ceeddata->xcorners, user->q0ceed, + dm, Qloc, Q, ceeddata->Erestrictq, + &ctxSetup, 0.0); + + MPI_Comm_rank(comm, &rank); + if (!rank) {ierr = PetscMkdir(user->outputfolder); CHKERRQ(ierr);} + // Gather initial Q values + // In case of continuation of simulation, set up initial values from binary file + if (contsteps) { // continue from existent solution + PetscViewer viewer; + char filepath[PETSC_MAX_PATH_LEN]; + // Read input + ierr = PetscSNPrintf(filepath, sizeof filepath, "%s/ns-solution.bin", + user->outputfolder); + CHKERRQ(ierr); + ierr = PetscViewerBinaryOpen(comm, filepath, FILE_MODE_READ, &viewer); + CHKERRQ(ierr); + ierr = VecLoad(Q, viewer); CHKERRQ(ierr); + ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr); + } + ierr = DMRestoreLocalVector(dm, &Qloc); CHKERRQ(ierr); + + // Set up the MatShell for the associated Jacobian operator + ierr = MatCreateShell(PETSC_COMM_SELF, 3*odofs, 3*odofs, PETSC_DETERMINE, + PETSC_DETERMINE, user, &J); CHKERRQ(ierr); + // Set the MatShell user context +// ierr = MatShellSetContext(J, user); CHKERRQ(ierr); + // Set the MatShell operation needed for the Jacobian + ierr = MatShellSetOperation(J, MATOP_MULT, + (void (*)(void))ApplyJacobian_SW); CHKERRQ(ierr); + + // Set up the MatShell for the associated Jacobian preconditioning operator +// MatCreateShell(PETSC_COMM_SELF, lsize, lsize, PETSC_DETERMINE, +// PETSC_DETERMINE, (void*)&user, &Jpre); +// MatShellSetOperation(Jpre, MATOP_MATMAT_MULT, (void(*)(void))PreJacobianProductMat); + + // Create and setup TS + ierr = TSCreate(comm, &ts); CHKERRQ(ierr); + ierr = TSSetDM(ts, dm); CHKERRQ(ierr); + ierr = TSSetType(ts,TSARKIMEX); CHKERRQ(ierr); + ierr = TSSetIFunction(ts, NULL, FormIFunction_SW, &user); CHKERRQ(ierr); + ierr = TSSetRHSFunction(ts, NULL, FormRHSFunction_SW, &user); CHKERRQ(ierr); + // TODO: Check TSSetIJacobian + ierr = DMSetMatType(dm, MATSHELL); CHKERRQ(ierr); + ierr = TSSetIJacobian(ts, J, J, FormJacobian_SW, &user); CHKERRQ(ierr); + + // Other TS options + ierr = TSSetMaxTime(ts, 500. * units->second); CHKERRQ(ierr); + ierr = TSSetExactFinalTime(ts, TS_EXACTFINALTIME_STEPOVER); CHKERRQ(ierr); + ierr = TSSetTimeStep(ts, 1.e-2 * units->second); CHKERRQ(ierr); + if (test) {ierr = TSSetMaxSteps(ts, 1); CHKERRQ(ierr);} + ierr = TSGetAdapt(ts, &adapt); CHKERRQ(ierr); + ierr = TSAdaptSetStepLimits(adapt, + 1.e-12 * units->second, + 1.e2 * units->second); CHKERRQ(ierr); + ierr = TSSetFromOptions(ts); CHKERRQ(ierr); + if (!contsteps) { // print initial condition + if (!test) { + ierr = TSMonitor_SW(ts, 0, 0., Q, user); CHKERRQ(ierr); + } + } else { // continue from time of last output + PetscReal time; + PetscInt count; + PetscViewer viewer; + char filepath[PETSC_MAX_PATH_LEN]; + ierr = PetscSNPrintf(filepath, sizeof filepath, "%s/ns-time.bin", + user->outputfolder); CHKERRQ(ierr); + ierr = PetscViewerBinaryOpen(comm, filepath, FILE_MODE_READ, &viewer); + CHKERRQ(ierr); + ierr = PetscViewerBinaryRead(viewer, &time, 1, &count, PETSC_REAL); + CHKERRQ(ierr); + ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr); + ierr = TSSetTime(ts, time * user->units->second); CHKERRQ(ierr); + } + if (!test) { + ierr = TSMonitorSet(ts, TSMonitor_SW, user, NULL); CHKERRQ(ierr); + } + + // Solve + start = MPI_Wtime(); + ierr = PetscBarrier((PetscObject)ts); CHKERRQ(ierr); + ierr = TSSolve(ts, Q); CHKERRQ(ierr); + cpu_time_used = MPI_Wtime() - start; + ierr = TSGetSolveTime(ts,&ftime); CHKERRQ(ierr); + ierr = MPI_Allreduce(MPI_IN_PLACE, &cpu_time_used, 1, MPI_DOUBLE, MPI_MIN, + comm); CHKERRQ(ierr); + if (!test) { + ierr = PetscPrintf(PETSC_COMM_WORLD, + "Time taken for solution: %g\n", + (double)cpu_time_used); CHKERRQ(ierr); + } + + // Output Statistics + ierr = TSGetStepNumber(ts,&steps); CHKERRQ(ierr); + if (!test) { + ierr = PetscPrintf(PETSC_COMM_WORLD, + "Time integrator took %D time steps to reach final time %g\n", + steps,(double)ftime); CHKERRQ(ierr); + } + + + // Clean up libCEED + CeedVectorDestroy(&ceeddata->qdata); + CeedVectorDestroy(&user->qceed); + CeedVectorDestroy(&user->fceed); + CeedVectorDestroy(&user->gceed); + CeedVectorDestroy(&user->jceed); + CeedVectorDestroy(&ceeddata->xcorners); + CeedBasisDestroy(&ceeddata->basisq); + CeedBasisDestroy(&ceeddata->basisx); + CeedElemRestrictionDestroy(&ceeddata->Erestrictq); + CeedElemRestrictionDestroy(&ceeddata->Erestrictx); + CeedElemRestrictionDestroy(&ceeddata->Erestrictqdi); + CeedQFunctionDestroy(&ceeddata->qf_setup); + CeedQFunctionDestroy(&ceeddata->qf_ics); + CeedQFunctionDestroy(&ceeddata->qf_explicit); + CeedQFunctionDestroy(&ceeddata->qf_implicit); + CeedQFunctionDestroy(&ceeddata->qf_jacobian); + CeedOperatorDestroy(&ceeddata->op_setup); + CeedOperatorDestroy(&ceeddata->op_ics); + CeedOperatorDestroy(&ceeddata->op_explicit); + CeedOperatorDestroy(&ceeddata->op_implicit); + CeedOperatorDestroy(&ceeddata->op_jacobian); + CeedDestroy(&ceed); + + // Clean up PETSc + ierr = VecDestroy(&Q); CHKERRQ(ierr); + ierr = VecDestroy(&user->M); CHKERRQ(ierr); + ierr = MatDestroy(&interpviz); CHKERRQ(ierr); + ierr = DMDestroy(&dmviz); CHKERRQ(ierr); + ierr = TSDestroy(&ts); CHKERRQ(ierr); + ierr = DMDestroy(&dm); CHKERRQ(ierr); + ierr = MatDestroy(&J);CHKERRQ(ierr); + ierr = PetscFree(units); CHKERRQ(ierr); + ierr = PetscFree(user); CHKERRQ(ierr); + return PetscFinalize(); +} diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c new file mode 100644 index 0000000000..b07f45a49d --- /dev/null +++ b/examples/fluids/shallow-water/src/setup.c @@ -0,0 +1,455 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +#include +#include +#include +#include +#include +#include +#include "../sw_headers.h" // Function prototytes +#include "../qfunctions/setup_geo.h" // Geometric factors +#include "../qfunctions/shallowwater.h" // Physics point-wise functions + +/// @file +/// Helper setup functions for shallow-water example using PETSc + +// ----------------------------------------------------------------------------- +// Auxiliary function to create PETSc FE space for a given degree +// ----------------------------------------------------------------------------- + +PetscErrorCode PetscFECreateByDegree(DM dm, PetscInt dim, PetscInt Nc, + PetscBool isSimplex, + const char prefix[], + PetscInt order, PetscFE *fem) { + PetscQuadrature q, fq; + DM K; + PetscSpace P; + PetscDualSpace Q; + PetscInt quadPointsPerEdge; + PetscBool tensor = isSimplex ? PETSC_FALSE : PETSC_TRUE; + PetscErrorCode ierr; + + PetscFunctionBeginUser; + /* Create space */ + ierr = PetscSpaceCreate(PetscObjectComm((PetscObject) dm), &P); CHKERRQ(ierr); + ierr = PetscObjectSetOptionsPrefix((PetscObject) P, prefix); CHKERRQ(ierr); + ierr = PetscSpacePolynomialSetTensor(P, tensor); CHKERRQ(ierr); + ierr = PetscSpaceSetFromOptions(P); CHKERRQ(ierr); + ierr = PetscSpaceSetNumComponents(P, Nc); CHKERRQ(ierr); + ierr = PetscSpaceSetNumVariables(P, dim); CHKERRQ(ierr); + ierr = PetscSpaceSetDegree(P, order, order); CHKERRQ(ierr); + ierr = PetscSpaceSetUp(P); CHKERRQ(ierr); + ierr = PetscSpacePolynomialGetTensor(P, &tensor); CHKERRQ(ierr); + /* Create dual space */ + ierr = PetscDualSpaceCreate(PetscObjectComm((PetscObject) dm), &Q); + CHKERRQ(ierr); + ierr = PetscDualSpaceSetType(Q,PETSCDUALSPACELAGRANGE); CHKERRQ(ierr); + ierr = PetscObjectSetOptionsPrefix((PetscObject) Q, prefix); CHKERRQ(ierr); + ierr = PetscDualSpaceCreateReferenceCell(Q, dim, isSimplex, &K); CHKERRQ(ierr); + ierr = PetscDualSpaceSetDM(Q, K); CHKERRQ(ierr); + ierr = DMDestroy(&K); CHKERRQ(ierr); + ierr = PetscDualSpaceSetNumComponents(Q, Nc); CHKERRQ(ierr); + ierr = PetscDualSpaceSetOrder(Q, order); CHKERRQ(ierr); + ierr = PetscDualSpaceLagrangeSetTensor(Q, tensor); CHKERRQ(ierr); + ierr = PetscDualSpaceSetFromOptions(Q); CHKERRQ(ierr); + ierr = PetscDualSpaceSetUp(Q); CHKERRQ(ierr); + /* Create element */ + ierr = PetscFECreate(PetscObjectComm((PetscObject) dm), fem); CHKERRQ(ierr); + ierr = PetscObjectSetOptionsPrefix((PetscObject) *fem, prefix); CHKERRQ(ierr); + ierr = PetscFESetFromOptions(*fem); CHKERRQ(ierr); + ierr = PetscFESetBasisSpace(*fem, P); CHKERRQ(ierr); + ierr = PetscFESetDualSpace(*fem, Q); CHKERRQ(ierr); + ierr = PetscFESetNumComponents(*fem, Nc); CHKERRQ(ierr); + ierr = PetscFESetUp(*fem); CHKERRQ(ierr); + ierr = PetscSpaceDestroy(&P); CHKERRQ(ierr); + ierr = PetscDualSpaceDestroy(&Q); CHKERRQ(ierr); + /* Create quadrature */ + quadPointsPerEdge = PetscMax(order + 1,1); + if (isSimplex) { + ierr = PetscDTStroudConicalQuadrature(dim, 1, quadPointsPerEdge, -1.0, 1.0, + &q); CHKERRQ(ierr); + ierr = PetscDTStroudConicalQuadrature(dim-1, 1, quadPointsPerEdge, -1.0, 1.0, + &fq); CHKERRQ(ierr); + } else { + ierr = PetscDTGaussTensorQuadrature(dim, 1, quadPointsPerEdge, -1.0, 1.0, + &q); CHKERRQ(ierr); + ierr = PetscDTGaussTensorQuadrature(dim-1, 1, quadPointsPerEdge, -1.0, 1.0, + &fq); CHKERRQ(ierr); + } + ierr = PetscFESetQuadrature(*fem, q); CHKERRQ(ierr); + ierr = PetscFESetFaceQuadrature(*fem, fq); CHKERRQ(ierr); + ierr = PetscQuadratureDestroy(&q); CHKERRQ(ierr); + ierr = PetscQuadratureDestroy(&fq); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// Auxiliary function to setup DM FE space and info +// ----------------------------------------------------------------------------- + +PetscErrorCode SetupDM(DM dm, PetscInt degree, PetscInt ncompq, + PetscInt dim) { + PetscErrorCode ierr; + + PetscFunctionBeginUser; + { + // Configure the finite element space + PetscFE fe; + PetscInt ncompq = 5; + ierr = PetscFECreateByDegree(dm, dim, ncompq, PETSC_FALSE, NULL, degree, + &fe); + ierr = PetscObjectSetName((PetscObject)fe, "Q"); CHKERRQ(ierr); + ierr = DMAddField(dm,NULL,(PetscObject)fe); CHKERRQ(ierr); + ierr = DMCreateDS(dm); CHKERRQ(ierr); + + ierr = DMPlexSetClosurePermutationTensor(dm, PETSC_DETERMINE, NULL); + CHKERRQ(ierr); + ierr = PetscFEDestroy(&fe); CHKERRQ(ierr); + } + { + // Empty name for conserved field (because there is only one field) + PetscSection section; + ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); + ierr = PetscSectionSetFieldName(section, 0, ""); CHKERRQ(ierr); + ierr = PetscSectionSetComponentName(section, 0, 0, "u_lambda"); + CHKERRQ(ierr); + ierr = PetscSectionSetComponentName(section, 0, 1, "u_theta"); + CHKERRQ(ierr); + ierr = PetscSectionSetComponentName(section, 0, 2, "h"); + CHKERRQ(ierr); + ierr = PetscSectionSetComponentName(section, 0, 3, "h_s"); + CHKERRQ(ierr); + ierr = PetscSectionSetComponentName(section, 0, 4, "H_0"); + CHKERRQ(ierr); + } + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// PETSc sphere auxiliary function +// ----------------------------------------------------------------------------- + +// Utility function taken from petsc/src/dm/impls/plex/examples/tutorials/ex7.c +PetscErrorCode ProjectToUnitSphere(DM dm) { + Vec coordinates; + PetscScalar *coords; + PetscInt Nv, v, dim, d; + PetscErrorCode ierr; + + PetscFunctionBeginUser; + ierr = DMGetCoordinatesLocal(dm, &coordinates); CHKERRQ(ierr); + ierr = VecGetLocalSize(coordinates, &Nv); CHKERRQ(ierr); + ierr = VecGetBlockSize(coordinates, &dim); CHKERRQ(ierr); + Nv /= dim; + ierr = VecGetArray(coordinates, &coords); CHKERRQ(ierr); + for (v = 0; v < Nv; ++v) { + PetscReal r = 0.0; + + for (d = 0; d < dim; ++d) r += PetscSqr(PetscRealPart(coords[v*dim+d])); + r = PetscSqrtReal(r); + for (d = 0; d < dim; ++d) coords[v*dim+d] /= r; + } + ierr = VecRestoreArray(coordinates, &coords); CHKERRQ(ierr); + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// Auxiliary function to define CEED restrictions from DMPlex data +// ----------------------------------------------------------------------------- + +PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, + CeedInt ncomp, + CeedElemRestriction *Erestrict) { + PetscInt ierr; + PetscInt c, cStart, cEnd, nelem, nnodes, *erestrict, eoffset; + PetscSection section; + Vec Uloc; + + PetscFunctionBegin; + + // Get Nelem + ierr = DMGetSection(dm, §ion); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(dm, 0, &cStart,& cEnd); CHKERRQ(ierr); + nelem = cEnd - cStart; + + // Get indices + ierr = PetscMalloc1(nelem*P*P, &erestrict); CHKERRQ(ierr); + for (c=cStart, eoffset = 0; cq0ceed = q0ceed; + + // Create the Q-Function that builds the quadrature data + CeedQFunctionCreateInterior(ceed, 1, SetupGeo, SetupGeo_loc, &qf_setup); + CeedQFunctionAddInput(qf_setup, "x", ncompx, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_setup, "dx", ncompx*topodim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_setup, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddOutput(qf_setup, "qdata", qdatasize, CEED_EVAL_NONE); + + // Create the Q-Function that sets the ICs of the operator + CeedQFunctionCreateInterior(ceed, 1, ICsSW, ICsSW_loc, &qf_ics); + CeedQFunctionAddInput(qf_ics, "x", ncompx, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_ics, "q0", ncompq, CEED_EVAL_NONE); + + // Create the Q-Function that defines the explicit part of the PDE operator + CeedQFunctionCreateInterior(ceed, 1, SWExplicit, SWExplicit_loc, + &qf_explicit); + CeedQFunctionAddInput(qf_explicit, "q", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_explicit, "dq", ncompq*topodim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_explicit, "qdata", qdatasize, CEED_EVAL_NONE); + CeedQFunctionAddInput(qf_explicit, "x", ncompx, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_explicit, "v", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_explicit, "dv", ncompq*topodim, CEED_EVAL_GRAD); + + // Create the Q-Function that defines the implicit part of the PDE operator + CeedQFunctionCreateInterior(ceed, 1, SWImplicit, SWImplicit_loc, + &qf_implicit); + CeedQFunctionAddInput(qf_implicit, "q", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_implicit, "dq", ncompq*topodim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_implicit, "qdot", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_implicit, "qdata", qdatasize, CEED_EVAL_NONE); + CeedQFunctionAddInput(qf_implicit, "x", ncompx, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_implicit, "v", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_implicit, "dv", ncompq*topodim, CEED_EVAL_GRAD); + + // Create the Q-Function that defines the action of the Jacobian operator + CeedQFunctionCreateInterior(ceed, 1, SWJacobian, SWJacobian_loc, + &qf_jacobian); + CeedQFunctionAddInput(qf_jacobian, "q", 3, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_jacobian, "deltaq", 3, CEED_EVAL_NONE); + CeedQFunctionAddInput(qf_jacobian, "qdata", 10, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_jacobian, "dv", 3, CEED_EVAL_GRAD); + + // Create the operator that builds the quadrature data for the operator + CeedOperatorCreate(ceed, qf_setup, NULL, NULL, &op_setup); + CeedOperatorSetField(op_setup, "x", Erestrictx, basisx, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup, "dx", Erestrictx, basisx, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup, "weight", CEED_ELEMRESTRICTION_NONE, basisx, + CEED_VECTOR_NONE); + CeedOperatorSetField(op_setup, "qdata", Erestrictqdi, + CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + + // Create the operator that sets the ICs + CeedOperatorCreate(ceed, qf_ics, NULL, NULL, &op_ics); + CeedOperatorSetField(op_ics, "x", Erestrictx, basisxc, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_ics, "q0", Erestrictq, + CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + + CeedElemRestrictionCreateVector(Erestrictq, &user->qceed, NULL); + CeedElemRestrictionCreateVector(Erestrictq, &user->qdotceed, NULL); + CeedElemRestrictionCreateVector(Erestrictq, &user->gceed, NULL); + + // Create the explicit part of the PDE operator + CeedOperatorCreate(ceed, qf_explicit, NULL, NULL, &op_explicit); + CeedOperatorSetField(op_explicit, "q", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_explicit, "dq", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_explicit, "qdata", Erestrictq, + CEED_BASIS_COLLOCATED, qdata); + CeedOperatorSetField(op_explicit, "x", Erestrictx, basisx, xcorners); + CeedOperatorSetField(op_explicit, "v", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_explicit, "dv", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + user->op_explicit = op_explicit; + + // Create the implicit part of the PDE operator + CeedOperatorCreate(ceed, qf_implicit, NULL, NULL, &op_implicit); + CeedOperatorSetField(op_implicit, "q", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_implicit, "dq", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_implicit, "qdot", Erestrictq, basisq, user->qdotceed); + CeedOperatorSetField(op_implicit, "qdata", Erestrictq, + CEED_BASIS_COLLOCATED, qdata); + CeedOperatorSetField(op_implicit, "x", Erestrictx, basisx, xcorners); + CeedOperatorSetField(op_implicit, "v", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_implicit, "dv", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + user->op_implicit = op_implicit; + + // Create the Jacobian of the PDE operator + CeedOperatorCreate(ceed, qf_jacobian, NULL, NULL, &op_jacobian); + CeedOperatorSetField(op_jacobian, "q", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "deltaq", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); // TO DO: Check restriction and basis for delta q + CeedOperatorSetField(op_jacobian, "qdata", Erestrictqdi, + CEED_BASIS_COLLOCATED, qdata); + CeedOperatorSetField(op_jacobian, "dv", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + user->op_jacobian = op_jacobian; + + CeedQFunctionSetContext(qf_ics, &ctxSetup, sizeof ctxSetup); + + // Set up the libCEED context + CeedScalar ctxSWICs[3] = {ctxSetup->u0, ctxSetup->v0, + ctxSetup->h0}; + CeedQFunctionSetContext(qf_ics, &ctxSWICs, sizeof ctxSWICs); + CeedScalar ctxSWExplicit = ctxSetup->f; + CeedQFunctionSetContext(qf_explicit, &ctxSWExplicit, sizeof ctxSWExplicit); + CeedScalar ctxSWImplicit = ctxSetup->g; + CeedQFunctionSetContext(qf_implicit, &ctxSWImplicit, sizeof ctxSWImplicit); + // same context for Jacobian qfunction + CeedQFunctionSetContext(qf_jacobian, &ctxSWImplicit, sizeof ctxSWImplicit); + + // Save libCEED data required for level // TODO: check how many of these are really needed outside + data->basisx = basisx; + data->basisq = basisq; + data->Erestrictx = Erestrictx; + data->Erestrictq = Erestrictq; + data->Erestrictqdi = Erestrictqdi; + data->qf_setup = qf_setup; + data->qf_ics = qf_ics; + data->qf_explicit = qf_explicit; + data->qf_implicit = qf_implicit; + data->qf_jacobian = qf_jacobian; + data->op_setup = op_setup; + data->op_ics = op_ics; + data->op_explicit = op_explicit; + data->op_implicit = op_implicit; + data->op_jacobian = op_jacobian; + data->qdata = qdata; + data->xcorners = xcorners; + + PetscFunctionReturn(0); +} diff --git a/examples/fluids/shallow-water/src/wrappers.c b/examples/fluids/shallow-water/src/wrappers.c new file mode 100644 index 0000000000..417841d546 --- /dev/null +++ b/examples/fluids/shallow-water/src/wrappers.c @@ -0,0 +1,381 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// libCEED auxiliary wrapper functions for shallow-water example using PETSc + +#include "../sw_headers.h" +#include "../qfunctions/mass.h" + +// ----------------------------------------------------------------------------- +// This froms the RHS of the IMEX ODE, given as F(t,Q,Q_t) = G(t,Q) +// This function takes in a state vector Q and writes into G +// ----------------------------------------------------------------------------- + +PetscErrorCode FormRHSFunction_SW(TS ts, PetscReal t, Vec Q, Vec G, + void *userData) { + PetscErrorCode ierr; + User user = *(User *)userData; + PetscScalar *q, *g; + Vec Qloc, Gloc; + + // Global-to-local + PetscFunctionBeginUser; + ierr = DMGetLocalVector(user->dm, &Qloc); CHKERRQ(ierr); + ierr = DMGetLocalVector(user->dm, &Gloc); CHKERRQ(ierr); + ierr = VecZeroEntries(Qloc); CHKERRQ(ierr); + ierr = DMGlobalToLocal(user->dm, Q, INSERT_VALUES, Qloc); CHKERRQ(ierr); + ierr = VecZeroEntries(Gloc); CHKERRQ(ierr); + + // Ceed Vectors + ierr = VecGetArrayRead(Qloc, (const PetscScalar **)&q); CHKERRQ(ierr); + ierr = VecGetArray(Gloc, &g); CHKERRQ(ierr); + CeedVectorSetArray(user->qceed, CEED_MEM_HOST, CEED_USE_POINTER, q); + CeedVectorSetArray(user->gceed, CEED_MEM_HOST, CEED_USE_POINTER, g); + + // Apply CEED operator + CeedOperatorApply(user->op_explicit, user->qceed, user->gceed, + CEED_REQUEST_IMMEDIATE); + + // Restore vectors + ierr = VecRestoreArrayRead(Qloc, (const PetscScalar **)&q); CHKERRQ(ierr); + ierr = VecRestoreArray(Gloc, &g); CHKERRQ(ierr); + + ierr = VecZeroEntries(G); CHKERRQ(ierr); + ierr = DMLocalToGlobal(user->dm, Gloc, ADD_VALUES, G); CHKERRQ(ierr); + + // Inverse of the lumped mass matrix + ierr = VecPointwiseMult(G, G, user->M); // M is Minv + CHKERRQ(ierr); + + ierr = DMRestoreLocalVector(user->dm, &Qloc); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(user->dm, &Gloc); CHKERRQ(ierr); + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// This forms the LHS of the IMEX ODE, given as F(t,Q,Qdot) = G(t,Q) +// This function takes in the state vector Q and its derivative Qdot +// and writes into F +// ----------------------------------------------------------------------------- + +PetscErrorCode FormIFunction_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, + Vec F, void *userData) { + PetscErrorCode ierr; + User user = *(User *)userData; + const PetscScalar *q, *qdot; + PetscScalar *f; + Vec Qloc, Qdotloc, Floc; + + // Global-to-local + PetscFunctionBeginUser; + ierr = DMGetLocalVector(user->dm, &Qloc); CHKERRQ(ierr); + ierr = DMGetLocalVector(user->dm, &Qdotloc); CHKERRQ(ierr); + ierr = DMGetLocalVector(user->dm, &Floc); CHKERRQ(ierr); + ierr = VecZeroEntries(Qloc); CHKERRQ(ierr); + ierr = DMGlobalToLocal(user->dm, Q, INSERT_VALUES, Qloc); CHKERRQ(ierr); + ierr = VecZeroEntries(Qdotloc); CHKERRQ(ierr); + ierr = DMGlobalToLocal(user->dm, Qdot, INSERT_VALUES, Qdotloc); CHKERRQ(ierr); + ierr = VecZeroEntries(Floc); CHKERRQ(ierr); + + // Ceed Vectors + ierr = VecGetArrayRead(Qloc, &q); CHKERRQ(ierr); + ierr = VecGetArrayRead(Qdotloc, &qdot); CHKERRQ(ierr); + ierr = VecGetArray(Floc, &f); CHKERRQ(ierr); + CeedVectorSetArray(user->qceed, CEED_MEM_HOST, CEED_USE_POINTER, + (PetscScalar *)q); + CeedVectorSetArray(user->qdotceed, CEED_MEM_HOST, CEED_USE_POINTER, + (PetscScalar *)qdot); + CeedVectorSetArray(user->fceed, CEED_MEM_HOST, CEED_USE_POINTER, f); + + // Apply CEED operator + CeedOperatorApply(user->op_implicit, user->qceed, user->fceed, + CEED_REQUEST_IMMEDIATE); + + // Restore vectors + ierr = VecRestoreArrayRead(Qloc, &q); CHKERRQ(ierr); + ierr = VecRestoreArrayRead(Qdotloc, &qdot); CHKERRQ(ierr); + ierr = VecRestoreArray(Floc, &f); CHKERRQ(ierr); + + ierr = VecZeroEntries(F); CHKERRQ(ierr); + ierr = DMLocalToGlobal(user->dm, Floc, ADD_VALUES, F); CHKERRQ(ierr); + + ierr = DMRestoreLocalVector(user->dm, &Qloc); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(user->dm, &Qdotloc); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(user->dm, &Floc); CHKERRQ(ierr); + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// User provided Jacobian = dF/dQ + sigma dF/dQdot +// ----------------------------------------------------------------------------- + +PetscErrorCode FormJacobian_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, + PetscReal sigma, Mat J, Mat Jpre, + void *userData) { +// User user = *(User*)userData; +// PetscErrorCode ierr; + + PetscFunctionBeginUser; + // empty body for now + +// // Set the preconditioning for the Jacobian, Jpre, to be = sigma M +// ierr = VecScale(user->M, sigma); CHKERRQ(ierr); +// ierr = VecCopy(M, JPreVec); CHKERRQ(ierr); + +// // Set up the MatShell for the associated Jacobian operator +// MatCreateShell(PETSC_COMM_SELF, lsize, lsize, PETSC_DETERMINE, +// PETSC_DETERMINE, (void*)&user, &J); +// MatShellSetOperation(J, MATOP_MULT, (void(*)(void))JacobianProductMat); + +// // Set up the MatShell for the associated Jacobian preconditioning operator +// MatCreateShell(PETSC_COMM_SELF, lsize, lsize, PETSC_DETERMINE, +// PETSC_DETERMINE, (void*)&user, &Jpre); +// MatShellSetOperation(Jpre, MATOP_MATMAT_MULT, (void(*)(void))PreJacobianProductMat); + + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// User provided wrapper function for MATOP_MULT MatShellOperation in PETSc +// for action of Jacobian calculation. Uses libCEED to compute the +// matrix-vector product: +// Jvec = mat*Q + +// Input Parameters: +// mat - input matrix +// Q - input vector +// +// Output Parameters: +// Jvec - output vector +// ----------------------------------------------------------------------------- + +PetscErrorCode ApplyJacobian_SW(Mat mat, Vec Q, Vec JVec) { + User user; + PetscScalar *q, *j; + Vec Qloc, Jloc; + PetscErrorCode ierr; + + PetscFunctionBeginUser; + MatShellGetContext(mat, &user); + ierr = DMGetLocalVector(user->dm, &Qloc); CHKERRQ(ierr); + ierr = DMGetLocalVector(user->dm, &Jloc); CHKERRQ(ierr); + ierr = VecZeroEntries(Qloc); CHKERRQ(ierr); + // Global-to-local + ierr = DMGlobalToLocal(user->dm, Q, INSERT_VALUES, Qloc); CHKERRQ(ierr); + ierr = VecZeroEntries(Jloc); CHKERRQ(ierr); + + // Setup CEED vectors + ierr = VecGetArrayRead(Qloc, (const PetscScalar **)&q); CHKERRQ(ierr); + ierr = VecGetArray(Jloc, &j); CHKERRQ(ierr); + CeedVectorSetArray(user->qceed, CEED_MEM_HOST, CEED_USE_POINTER, q); + CeedVectorSetArray(user->jceed, CEED_MEM_HOST, CEED_USE_POINTER, j); + + // Apply the CEED operator for the dF/dQ terms + CeedOperatorApply(user->op_jacobian, user->qceed, user->jceed, + CEED_REQUEST_IMMEDIATE); + CeedVectorSyncArray(user->jceed, CEED_MEM_HOST); + + // Restore PETSc vectors + ierr = VecRestoreArrayRead(Qloc, (const PetscScalar **)&q); + CHKERRQ(ierr); + ierr = VecRestoreArray(Jloc, &j); CHKERRQ(ierr); + + // Local-to-global + ierr = VecZeroEntries(JVec); CHKERRQ(ierr); + ierr = DMLocalToGlobal(user->dm, Jloc, ADD_VALUES, JVec); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} + + +// ----------------------------------------------------------------------------- +// TS Monitor to print output +// ----------------------------------------------------------------------------- + +PetscErrorCode TSMonitor_SW(TS ts, PetscInt stepno, PetscReal time, + Vec Q, void *ctx) { + User user = ctx; + Vec Qloc; + char filepath[PETSC_MAX_PATH_LEN]; + PetscViewer viewer; + PetscErrorCode ierr; + + // Set up output + PetscFunctionBeginUser; + // Print every 'outputfreq' steps + if (stepno % user->outputfreq != 0) + PetscFunctionReturn(0); + ierr = DMGetLocalVector(user->dm, &Qloc); CHKERRQ(ierr); + ierr = PetscObjectSetName((PetscObject)Qloc, "StateVec"); CHKERRQ(ierr); + ierr = VecZeroEntries(Qloc); CHKERRQ(ierr); + ierr = DMGlobalToLocal(user->dm, Q, INSERT_VALUES, Qloc); CHKERRQ(ierr); + + // Output + ierr = PetscSNPrintf(filepath, sizeof filepath, "%s/swe-%03D.vtu", + user->outputfolder, stepno + user->contsteps); + CHKERRQ(ierr); + ierr = PetscViewerVTKOpen(PetscObjectComm((PetscObject)Q), filepath, + FILE_MODE_WRITE, &viewer); CHKERRQ(ierr); + ierr = VecView(Qloc, viewer); CHKERRQ(ierr); + if (user->dmviz) { + Vec Qrefined, Qrefined_loc; + char filepath_refined[PETSC_MAX_PATH_LEN]; + PetscViewer viewer_refined; + + ierr = DMGetGlobalVector(user->dmviz, &Qrefined); CHKERRQ(ierr); + ierr = DMGetLocalVector(user->dmviz, &Qrefined_loc); CHKERRQ(ierr); + ierr = PetscObjectSetName((PetscObject)Qrefined_loc, "Refined"); + CHKERRQ(ierr); + ierr = MatInterpolate(user->interpviz, Q, Qrefined); CHKERRQ(ierr); + ierr = VecZeroEntries(Qrefined_loc); CHKERRQ(ierr); + ierr = DMGlobalToLocal(user->dmviz, Qrefined, INSERT_VALUES, Qrefined_loc); + CHKERRQ(ierr); + ierr = PetscSNPrintf(filepath_refined, sizeof filepath_refined, + "%s/swe-refined-%03D.vtu", + user->outputfolder, stepno + user->contsteps); + CHKERRQ(ierr); + ierr = PetscViewerVTKOpen(PetscObjectComm((PetscObject)Qrefined), + filepath_refined, + FILE_MODE_WRITE, &viewer_refined); CHKERRQ(ierr); + ierr = VecView(Qrefined_loc, viewer_refined); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(user->dmviz, &Qrefined_loc); CHKERRQ(ierr); + ierr = DMRestoreGlobalVector(user->dmviz, &Qrefined); CHKERRQ(ierr); + ierr = PetscViewerDestroy(&viewer_refined); CHKERRQ(ierr); + } + ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(user->dm, &Qloc); CHKERRQ(ierr); + + // Save data in a binary file for continuation of simulations + ierr = PetscSNPrintf(filepath, sizeof filepath, "%s/swe-solution.bin", + user->outputfolder); CHKERRQ(ierr); + ierr = PetscViewerBinaryOpen(user->comm, filepath, FILE_MODE_WRITE, &viewer); + CHKERRQ(ierr); + ierr = VecView(Q, viewer); CHKERRQ(ierr); + ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr); + + // Save time stamp + // Dimensionalize time back + time /= user->units->second; + ierr = PetscSNPrintf(filepath, sizeof filepath, "%s/swe-time.bin", + user->outputfolder); CHKERRQ(ierr); + ierr = PetscViewerBinaryOpen(user->comm, filepath, FILE_MODE_WRITE, &viewer); + CHKERRQ(ierr); + #if PETSC_VERSION_GE(3,13,0) + ierr = PetscViewerBinaryWrite(viewer, &time, 1, PETSC_REAL); + #else + ierr = PetscViewerBinaryWrite(viewer, &time, 1, PETSC_REAL, true); + #endif + CHKERRQ(ierr); + ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// Auxiliary function to apply the ICs and eliminate repeated values in initial +// state vector, arising from restriction +// ----------------------------------------------------------------------------- + +PetscErrorCode ICs_FixMultiplicity(CeedOperator op_ics, + CeedVector xcorners, CeedVector q0ceed, DM dm, Vec Qloc, Vec Q, + CeedElemRestriction restrictq, PhysicsContext ctxSetup, CeedScalar time) { + PetscErrorCode ierr; + CeedVector multlvec; + Vec Multiplicity, MultiplicityLoc; + + ctxSetup->time = time; + ierr = VecZeroEntries(Qloc); CHKERRQ(ierr); + ierr = VectorPlacePetscVec(q0ceed, Qloc); CHKERRQ(ierr); + + // Apply IC operator + CeedOperatorApply(op_ics, xcorners, q0ceed, CEED_REQUEST_IMMEDIATE); + ierr = VecZeroEntries(Q); CHKERRQ(ierr); + ierr = DMLocalToGlobal(dm, Qloc, ADD_VALUES, Q); CHKERRQ(ierr); + + // Fix multiplicity for output of ICs + ierr = DMGetLocalVector(dm, &MultiplicityLoc); CHKERRQ(ierr); + CeedElemRestrictionCreateVector(restrictq, &multlvec, NULL); + ierr = VectorPlacePetscVec(multlvec, MultiplicityLoc); CHKERRQ(ierr); + CeedElemRestrictionGetMultiplicity(restrictq, multlvec); + CeedVectorDestroy(&multlvec); + ierr = DMGetGlobalVector(dm, &Multiplicity); CHKERRQ(ierr); + ierr = VecZeroEntries(Multiplicity); CHKERRQ(ierr); + ierr = DMLocalToGlobal(dm, MultiplicityLoc, ADD_VALUES, Multiplicity); + CHKERRQ(ierr); + ierr = VecPointwiseDivide(Q, Q, Multiplicity); CHKERRQ(ierr); + ierr = VecPointwiseDivide(Qloc, Qloc, MultiplicityLoc); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(dm, &MultiplicityLoc); CHKERRQ(ierr); + ierr = DMRestoreGlobalVector(dm, &Multiplicity); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// Auxiliary function to compute the lumped mass matrix +// ----------------------------------------------------------------------------- + +PetscErrorCode ComputeLumpedMassMatrix(Ceed ceed, DM dm, + CeedElemRestriction restrictq, CeedBasis basisq, + CeedElemRestriction restrictqdi, CeedVector qdata, Vec M) { + PetscErrorCode ierr; + CeedQFunction qf_mass; + CeedOperator op_mass; + CeedVector mceed; + Vec Mloc; + CeedInt ncompq, qdatasize; + + PetscFunctionBeginUser; + CeedElemRestrictionGetNumComponents(restrictq, &ncompq); + CeedElemRestrictionGetNumComponents(restrictqdi, &qdatasize); + // Create the Q-function that defines the action of the mass operator + CeedQFunctionCreateInterior(ceed, 1, Mass, Mass_loc, &qf_mass); + CeedQFunctionAddInput(qf_mass, "q", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_mass, "qdata", qdatasize, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_mass, "v", ncompq, CEED_EVAL_INTERP); + + // Create the mass operator + CeedOperatorCreate(ceed, qf_mass, NULL, NULL, &op_mass); + CeedOperatorSetField(op_mass, "q", restrictq, basisq, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_mass, "qdata", restrictqdi, + CEED_BASIS_COLLOCATED, qdata); + CeedOperatorSetField(op_mass, "v", restrictq, basisq, CEED_VECTOR_ACTIVE); + + ierr = DMGetLocalVector(dm, &Mloc); CHKERRQ(ierr); + ierr = VecZeroEntries(Mloc); CHKERRQ(ierr); + CeedElemRestrictionCreateVector(restrictq, &mceed, NULL); + ierr = VectorPlacePetscVec(mceed, Mloc); CHKERRQ(ierr); + + { + // Compute a lumped mass matrix + CeedVector onesvec; + CeedElemRestrictionCreateVector(restrictq, &onesvec, NULL); + CeedVectorSetValue(onesvec, 1.0); + CeedOperatorApply(op_mass, onesvec, mceed, CEED_REQUEST_IMMEDIATE); + CeedVectorDestroy(&onesvec); + CeedOperatorDestroy(&op_mass); + CeedVectorDestroy(&mceed); + } + CeedQFunctionDestroy(&qf_mass); + + ierr = VecZeroEntries(M); CHKERRQ(ierr); + ierr = DMLocalToGlobal(dm, Mloc, ADD_VALUES, M); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(dm, &Mloc); CHKERRQ(ierr); + + // Invert diagonally lumped mass vector for RHS function + ierr = VecReciprocal(M); CHKERRQ(ierr); + PetscFunctionReturn(0); +} diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h new file mode 100644 index 0000000000..4412dbcfd5 --- /dev/null +++ b/examples/fluids/shallow-water/sw_headers.h @@ -0,0 +1,180 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +#ifndef sw_headers_h +#define sw_headers_h + +#include +#include +#include +#include +#include +#include + +#ifndef physics_context_struct +#define physics_context_struct +typedef struct { + CeedScalar u0; + CeedScalar v0; + CeedScalar h0; + CeedScalar f; + CeedScalar g; + CeedScalar time; +} PhysicsContext_s; +typedef PhysicsContext_s *PhysicsContext; +#endif // physics_context_struct + +// Problem specific data +typedef struct { + CeedInt topodim, qdatasize; + CeedQFunctionUser setup, ics, apply_explfunction, apply_implfunction; + const char *setup_loc, *ics_loc, *apply_explfunction_loc, + *apply_implfunction_loc; + const bool non_zero_time; +} problemData; + +// PETSc user data +typedef struct User_ *User; +typedef struct Units_ *Units; + +struct User_ { + MPI_Comm comm; + PetscInt outputfreq; + DM dm; + DM dmviz; + Mat interpviz; + Ceed ceed; + Units units; + CeedVector qceed, q0ceed, qdotceed, fceed, gceed, jceed; + CeedOperator op_explicit, op_implicit, op_jacobian; + Vec M; + char outputfolder[PETSC_MAX_PATH_LEN]; + PetscInt contsteps; +}; + +struct Units_ { + // fundamental units + PetscScalar meter; + PetscScalar second; + // derived unit + PetscScalar mpersquareds; +}; + +// ----------------------------------------------------------------------------- +// libCEED Data Struct +// ----------------------------------------------------------------------------- + +// libCEED data struct +typedef struct CeedData_ *CeedData; +struct CeedData_ { + Ceed ceed; + CeedBasis basisx, basisxc, basisq; + CeedElemRestriction Erestrictx, Erestrictq, Erestrictqdi; + CeedQFunction qf_setup, qf_mass, qf_ics, qf_explicit, qf_implicit, + qf_jacobian; + CeedOperator op_setup, op_mass, op_ics, op_explicit, op_implicit, op_jacobian; + CeedVector xcorners, xceed, qdata, q0ceed, mceed, hsceed, H0ceed; +}; + +// ----------------------------------------------------------------------------- +// Setup DM functions +// ----------------------------------------------------------------------------- + +// Auxiliary function to create PETSc FE space for a given degree +PetscErrorCode PetscFECreateByDegree(DM dm, PetscInt dim, PetscInt Nc, + PetscBool isSimplex, const char prefix[], + PetscInt order, PetscFE *fem); + +// Auxiliary function to setup DM FE space and info +PetscErrorCode SetupDM(DM dm, PetscInt degree, PetscInt ncompq, PetscInt dim); + +// ----------------------------------------------------------------------------- +// libCEED functions +// ----------------------------------------------------------------------------- + +// Auxiliary function to define CEED restrictions from DMPlex data +PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, CeedInt ncomp, + CeedElemRestriction *Erestrict); + +// Auxiliary function to set up libCEED objects for a given degree +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, + CeedInt topodim, CeedInt qextra, + PetscInt ncompx, PetscInt ncompq, + User user, CeedData data, + PhysicsContext ctx); + +// ----------------------------------------------------------------------------- +// RHS (Explicit part in time-stepper) function setup +// ----------------------------------------------------------------------------- + +// This forms the RHS of the IMEX ODE, given as F(t,Q,Q_t) = G(t,Q) +PetscErrorCode FormRHSFunction_SW(TS ts, PetscReal t, Vec Q, Vec G, + void *userData); + +// ----------------------------------------------------------------------------- +// Implicit part in time-stepper function setup +// ----------------------------------------------------------------------------- + +// This forms the LHS of the IMEX ODE, given as F(t,Q,Qdot) = G(t,Q) +PetscErrorCode FormIFunction_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, + Vec F, void *userData); + +// This forms the RHS of the IMEX ODE, given as F(t,Q,Q_t) = G(t,Q) +PetscErrorCode FormRHSFunction_SW(TS ts, PetscReal t, Vec Q, Vec G, + void *userData); + +// ----------------------------------------------------------------------------- +// Jacobian setup and apply +// ----------------------------------------------------------------------------- + +PetscErrorCode FormJacobian_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, + PetscReal sigma, Mat J, Mat Jpre, + void *userData); + +PetscErrorCode ApplyJacobian_SW(Mat mat, Vec Q, Vec JVec); + +// ----------------------------------------------------------------------------- +// TS Monitor to print output +// ----------------------------------------------------------------------------- + +PetscErrorCode TSMonitor_SW(TS ts, PetscInt stepno, PetscReal time, + Vec Q, void *ctx); + +// ----------------------------------------------------------------------------- +// Miscellaneous utility functions +// ----------------------------------------------------------------------------- + +// Utility function to project refined discrete mesh points onto the unit sphere +PetscErrorCode ProjectToUnitSphere(DM dm); + +// Auxiliary function to create a CeedVector from PetscVec of same size +PetscErrorCode CreateVectorFromPetscVec(Ceed ceed, Vec p, CeedVector *v); + +// Auxiliary function to place a PetscVec into a CeedVector of same size +PetscErrorCode VectorPlacePetscVec(CeedVector c, Vec p); + +// Auxiliary function to apply the ICs and eliminate repeated values in initial +// state vector, arising from restriction +PetscErrorCode ICs_FixMultiplicity(CeedOperator op_ics, + CeedVector xcorners, CeedVector q0ceed, DM dm, Vec Qloc, Vec Q, + CeedElemRestriction restrictq, PhysicsContext ctx, CeedScalar time); + +// Auxiliary function to compute the lumped mass matrix +PetscErrorCode ComputeLumpedMassMatrix(Ceed ceed, DM dm, + CeedElemRestriction restrictq, CeedBasis basisq, + CeedElemRestriction restrictqdi, CeedVector qdata, Vec M); + +#endif // sw_headers_h From b3a0691d65faacd49bdb1a240384e690678f5ba6 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Thu, 14 May 2020 18:16:56 -0600 Subject: [PATCH 03/37] SWE: support for PETSc-3.14 DMPlexGetClosureIndices and DMPlexRestoreClosureIndices --- examples/fluids/shallow-water/src/setup.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index b07f45a49d..76f5db6b06 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -14,6 +14,9 @@ // software, applications, hardware, advanced system engineering and early // testbed platforms, in support of the nation's exascale computing imperative. +/// @file +/// Helper setup functions for shallow-water example using PETSc + #include #include #include @@ -24,8 +27,10 @@ #include "../qfunctions/setup_geo.h" // Geometric factors #include "../qfunctions/shallowwater.h" // Physics point-wise functions -/// @file -/// Helper setup functions for shallow-water example using PETSc +#if PETSC_VERSION_LT(3,14,0) +# define DMPlexGetClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexGetClosureIndices(a,b,c,d,f,g,i) +# define DMPlexRestoreClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexRestoreClosureIndices(a,b,c,d,f,g,i) +#endif // ----------------------------------------------------------------------------- // Auxiliary function to create PETSc FE space for a given degree @@ -191,8 +196,9 @@ PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, ierr = PetscMalloc1(nelem*P*P, &erestrict); CHKERRQ(ierr); for (c=cStart, eoffset = 0; c Date: Wed, 3 Jun 2020 10:55:34 -0600 Subject: [PATCH 04/37] Add copyright statement, improve summary print and other small edits --- examples/fluids/README.md | 4 + examples/fluids/index.rst | 2 +- .../shallow-water/qfunctions/shallowwater.h | 30 ++-- examples/fluids/shallow-water/shallowwater.c | 137 +++++++++++++----- examples/fluids/shallow-water/src/setup.c | 24 +-- examples/fluids/shallow-water/sw_headers.h | 10 +- 6 files changed, 138 insertions(+), 69 deletions(-) diff --git a/examples/fluids/README.md b/examples/fluids/README.md index c8c23c18f3..0d3bc2623a 100644 --- a/examples/fluids/README.md +++ b/examples/fluids/README.md @@ -12,6 +12,8 @@ with different problem definitions according to the application of interest. Build by using +`cd navier-stokes` + `make` and run with @@ -282,6 +284,8 @@ The main shallow-water solver for libCEED is defined in [`shallowwater.c`](shall Build by using +`cd shallow-water` + `make` and run with diff --git a/examples/fluids/index.rst b/examples/fluids/index.rst index 9ebf1a30dc..a346807b62 100644 --- a/examples/fluids/index.rst +++ b/examples/fluids/index.rst @@ -3,7 +3,7 @@ Compressible Navier-Stokes mini-app ======================================== -This example is located in the subdirectory :file:`examples/fluids`. It solves +This example is located in the subdirectory :file:`examples/fluids/navier-stokes`. It solves the time-dependent Navier-Stokes equations of compressible gas dynamics in a static Eulerian three-dimensional frame using unstructured high-order finite/spectral element spatial discretizations and explicit or implicit high-order time-stepping (available in diff --git a/examples/fluids/shallow-water/qfunctions/shallowwater.h b/examples/fluids/shallow-water/qfunctions/shallowwater.h index 4d1ca26b6f..435f1952f2 100644 --- a/examples/fluids/shallow-water/qfunctions/shallowwater.h +++ b/examples/fluids/shallow-water/qfunctions/shallowwater.h @@ -34,7 +34,8 @@ typedef struct { CeedScalar u0; CeedScalar v0; CeedScalar h0; - CeedScalar f; + CeedScalar Omega; + CeedScalar R; CeedScalar g; CeedScalar time; } PhysicsContext_s; @@ -52,9 +53,9 @@ static inline int Exact_SW(CeedInt dim, CeedScalar time, const CeedScalar X[], // Context const PhysicsContext context = (PhysicsContext)ctx; - const CeedScalar u0 = context->u0; - const CeedScalar v0 = context->v0; - const CeedScalar h0 = context->h0; + const CeedScalar u0 = context->u0; + const CeedScalar v0 = context->v0; + const CeedScalar h0 = context->h0; // Setup // -- Coordinates @@ -120,7 +121,8 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*q)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], + const CeedScalar (*X)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], + (*q)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], (*dq)[5][CEED_Q_VLA] = (const CeedScalar(*)[5][CEED_Q_VLA])in[1], (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs @@ -129,14 +131,20 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // *INDENT-ON* // Context - const CeedScalar *context = (const CeedScalar *)ctx; - const CeedScalar f = context[0]; + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar Omega = context->Omega; + const CeedScalar R = context->R; CeedPragmaSIMD // Quadrature Point Loop for (CeedInt i=0; ig; CeedPragmaSIMD // Quadrature Point Loop @@ -294,8 +302,8 @@ CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar (*deltadvdX)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[1]; // *INDENT-ON* // Context - const CeedScalar *context = (const CeedScalar *)ctx; - const CeedScalar g = context[0]; + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar g = context->g; CeedPragmaSIMD // Quadrature Point Loop diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index bc70ab9fb5..46a207ccbb 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -1,12 +1,27 @@ -// libCEED + PETSc Example: Shallow-water equations +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// libCEED + PETSc Example: Shallow-water equations // // This example demonstrates a simple usage of libCEED with PETSc to solve the -// shallow-water equations on a cubed-sphere (i.e., a spherical surface -// tessellated by quadrilaterals, obtained by projecting the sides -// of a circumscribed cube onto a spherical surface). +// shallow-water equations on a cubed-sphere (i.e., a tensor-product discrete +// sphere, obtained by projecting a cube inscribed in a sphere onto the surface +// of the sphere). // -// The code is intentionally "raw", using only low-level communication -// primitives. +// The code uses higher level communication protocols in PETSc's DMPlex. // // Build with: // @@ -27,11 +42,6 @@ const char help[] = "Solve the shallow-water equations using PETSc and libCEED\n"; -#include -#include -#include -#include -#include #include "sw_headers.h" int main(int argc, char **argv) { @@ -57,11 +67,22 @@ int main(int argc, char **argv) { filename[PETSC_MAX_PATH_LEN]; Ceed ceed; CeedData ceeddata; - PetscScalar meter = 1e-2; // 1 meter in scaled length units - PetscScalar second = 1e-2; // 1 second in scaled time units - PetscScalar f = 0.0001; // mid-latitude Coriolis parameter - PetscScalar g = 9.81; // m/s^2 + CeedMemType memtyperequested; + PetscScalar meter = 1e-2; // 1 meter in scaled length units + PetscScalar second = 1e-2; // 1 second in scaled time units + PetscScalar Omega = 7.29212e-5; // Earth rotation rate (1/s) + PetscScalar g = 9.81; // gravitational acceleration (m/s^2) + PetscScalar R = 6.37122e6; // Earth radius (m) PetscScalar mpersquareds; + // Check PETSc CUDA support + PetscBool petschavecuda, setmemtyperequest = PETSC_FALSE; + // *INDENT-OFF* + #ifdef PETSC_HAVE_CUDA + petschavecuda = PETSC_TRUE; + #else + petschavecuda = PETSC_FALSE; + #endif + // *INDENT-ON* // Performance context double start, cpu_time_used; @@ -73,10 +94,6 @@ int main(int argc, char **argv) { ierr = PetscMalloc1(1, &user); CHKERRQ(ierr); ierr = PetscMalloc1(1, &units); CHKERRQ(ierr); - // Set up problem type command line option - //PetscFunctionListAdd(&icsflist, "sphere", &ICsSW); - //PetscFunctionListAdd(&qflist, "shallow-water", &SW); - // Parse command line options comm = PETSC_COMM_WORLD; ierr = PetscOptionsBegin(comm, NULL, "Shallow-water equations in PETSc with libCEED", @@ -110,8 +127,6 @@ int main(int argc, char **argv) { NULL, qextra, &qextra, NULL); CHKERRQ(ierr); ierr = PetscOptionsScalar("-g", "Gravitational acceleration", NULL, g, &g, NULL); CHKERRQ(ierr); - ierr = PetscOptionsScalar("-f", "Mid-latitude Coriolis parameter", - NULL, f, &f, NULL); CHKERRQ(ierr); PetscStrncpy(user->outputfolder, ".", 2); ierr = PetscOptionsString("-of", "Output folder", NULL, user->outputfolder, user->outputfolder, @@ -123,21 +138,28 @@ int main(int argc, char **argv) { simplex = PETSC_FALSE; ierr = PetscOptionsBool("-simplex", "Use simplices, or tensor product cells", NULL, simplex, &simplex, NULL); CHKERRQ(ierr); + memtyperequested = petschavecuda ? CEED_MEM_DEVICE : CEED_MEM_HOST; + ierr = PetscOptionsEnum("-memtype", + "CEED MemType requested", NULL, + memTypes, (PetscEnum)memtyperequested, + (PetscEnum *)&memtyperequested, &setmemtyperequest); + CHKERRQ(ierr); ierr = PetscOptionsEnd(); CHKERRQ(ierr); // Define derived units mpersquareds = meter / PetscSqr(second); // Scale variables to desired units - f /= second; + Omega /= second; g *= mpersquareds; // Set up the libCEED context - PhysicsContext_s ctxSetup = { + PhysicsContext_s ctx = { .u0 = 0., .v0 = 0., .h0 = .1, - .f = f, + .Omega = Omega, + .R = R, .g = g, .time = 0. }; @@ -216,25 +238,62 @@ int main(int argc, char **argv) { ierr = VecGetSize(Qloc, &lnodes); CHKERRQ(ierr); lnodes /= ncompq; + // Initialize CEED + CeedInit(ceedresource, &ceed); + // Set memtype + CeedMemType memtypebackend; + CeedGetPreferredMemType(ceed, &memtypebackend); + // Check memtype compatibility + if (!setmemtyperequest) + memtyperequested = memtypebackend; + else if (!petschavecuda && memtyperequested == CEED_MEM_DEVICE) + SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_SUP_SYS, + "PETSc was not built with CUDA. " + "Requested MemType CEED_MEM_DEVICE is not supported.", NULL); + // Print grid information PetscInt numP = degree + 1, numQ = numP + qextra; PetscInt gdofs, odofs; { + const char *usedresource; + CeedGetResource(ceed, &usedresource); + int comm_size; + ierr = MPI_Comm_size(comm, &comm_size); CHKERRQ(ierr); + ierr = VecGetSize(Q, &gdofs); CHKERRQ(ierr); ierr = VecGetLocalSize(Q, &odofs); CHKERRQ(ierr); - ierr = MPI_Comm_size(comm, &comm_size); CHKERRQ(ierr); + VecType vectype; + ierr = VecGetType(Q, &vectype); CHKERRQ(ierr); + + PetscInt cStart, cEnd; + ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr); + if (!test) { ierr = PetscPrintf(comm, "\n-- CEED Shallow-water equations solver on the cubed-sphere -- libCEED + PETSc --\n" + " rank(s) : %d\n" + " PETSc:\n" + " PETSc Vec Type : %s\n" " libCEED:\n" " libCEED Backend : %s\n" + " libCEED Backend MemType : %s\n" + " libCEED User Requested MemType : %s\n" " FEM space:\n" - " Number of 1D Basis Nodes (p) : %d\n" - " Number of 1D Quadrature Points (q) : %d\n" - " Global FEM dofs: %D (%D owned) on %d rank(s)\n" - " Local FEM nodes: %D\n", - ceedresource, numP, numQ, gdofs, odofs, comm_size, lnodes); CHKERRQ(ierr); + " Number of 1D Basis Nodes (P) : %d\n" + " Number of 1D Quadrature Points (Q) : %d\n" + " Local Elements : %D\n" + " Global nodes : %D\n" + " Owned nodes : %D\n" + " DoFs per node : %D\n" + " Global DoFs : %D\n" + " Owned DoFs : %D\n", + comm_size, ceedresource, usedresource, + CeedMemTypes[memtypebackend], + (setmemtyperequest) ? + CeedMemTypes[memtyperequested] : "none", numP, numQ, + cEnd - cStart, gdofs/ncompq, odofs/ncompq, ncompq, + gdofs, odofs); CHKERRQ(ierr); } } @@ -242,13 +301,10 @@ int main(int argc, char **argv) { // Set up global mass vector ierr = VecDuplicate(Q, &user->M); CHKERRQ(ierr); - // Initialize CEED - CeedInit(ceedresource, &ceed); - // Setup libCEED's objects ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); ierr = SetupLibceed(dm, ceed, degree, topodim, qextra, - ncompx, ncompq, user, ceeddata, &ctxSetup); CHKERRQ(ierr); + ncompx, ncompq, user, ceeddata, &ctx); CHKERRQ(ierr); // Set up PETSc context // Set up units structure @@ -278,23 +334,24 @@ int main(int argc, char **argv) { CEED_REQUEST_IMMEDIATE); ierr = ComputeLumpedMassMatrix(ceed, dm, ceeddata->Erestrictq, ceeddata->basisq, ceeddata->Erestrictqdi, - ceeddata->qdata, - user->M); CHKERRQ(ierr); + ceeddata->qdata, user->M); CHKERRQ(ierr); // Apply IC operator and fix multiplicity of initial state vector ierr = ICs_FixMultiplicity(ceeddata->op_ics, ceeddata->xcorners, user->q0ceed, dm, Qloc, Q, ceeddata->Erestrictq, - &ctxSetup, 0.0); + &ctx, 0.0); CHKERRQ(ierr); MPI_Comm_rank(comm, &rank); - if (!rank) {ierr = PetscMkdir(user->outputfolder); CHKERRQ(ierr);} + if (!rank) { + ierr = PetscMkdir(user->outputfolder); CHKERRQ(ierr); + } // Gather initial Q values // In case of continuation of simulation, set up initial values from binary file if (contsteps) { // continue from existent solution PetscViewer viewer; char filepath[PETSC_MAX_PATH_LEN]; // Read input - ierr = PetscSNPrintf(filepath, sizeof filepath, "%s/ns-solution.bin", + ierr = PetscSNPrintf(filepath, sizeof filepath, "%s/swe-solution.bin", user->outputfolder); CHKERRQ(ierr); ierr = PetscViewerBinaryOpen(comm, filepath, FILE_MODE_READ, &viewer); @@ -305,7 +362,7 @@ int main(int argc, char **argv) { ierr = DMRestoreLocalVector(dm, &Qloc); CHKERRQ(ierr); // Set up the MatShell for the associated Jacobian operator - ierr = MatCreateShell(PETSC_COMM_SELF, 3*odofs, 3*odofs, PETSC_DETERMINE, + ierr = MatCreateShell(PETSC_COMM_SELF, 5*odofs, 5*odofs, PETSC_DETERMINE, PETSC_DETERMINE, user, &J); CHKERRQ(ierr); // Set the MatShell user context // ierr = MatShellSetContext(J, user); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 76f5db6b06..c9d39b0aa2 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -115,7 +115,6 @@ PetscErrorCode SetupDM(DM dm, PetscInt degree, PetscInt ncompq, { // Configure the finite element space PetscFE fe; - PetscInt ncompq = 5; ierr = PetscFECreateByDegree(dm, dim, ncompq, PETSC_FALSE, NULL, degree, &fe); ierr = PetscObjectSetName((PetscObject)fe, "Q"); CHKERRQ(ierr); @@ -272,7 +271,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt topodim, CeedInt qextra, PetscInt ncompx, PetscInt ncompq, User user, CeedData data, - PhysicsContext ctxSetup) { + PhysicsContext ctx) { int ierr; DM dmcoord; Vec Xloc; @@ -336,10 +335,10 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, // Create the Q-Function that defines the explicit part of the PDE operator CeedQFunctionCreateInterior(ceed, 1, SWExplicit, SWExplicit_loc, &qf_explicit); + CeedQFunctionAddInput(qf_explicit, "x", ncompx, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_explicit, "q", ncompq, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_explicit, "dq", ncompq*topodim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_explicit, "qdata", qdatasize, CEED_EVAL_NONE); - CeedQFunctionAddInput(qf_explicit, "x", ncompx, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_explicit, "v", ncompq, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_explicit, "dv", ncompq*topodim, CEED_EVAL_GRAD); @@ -385,13 +384,13 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, // Create the explicit part of the PDE operator CeedOperatorCreate(ceed, qf_explicit, NULL, NULL, &op_explicit); + CeedOperatorSetField(op_explicit, "x", Erestrictx, basisx, xcorners); CeedOperatorSetField(op_explicit, "q", Erestrictq, basisq, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_explicit, "dq", Erestrictq, basisq, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_explicit, "qdata", Erestrictq, CEED_BASIS_COLLOCATED, qdata); - CeedOperatorSetField(op_explicit, "x", Erestrictx, basisx, xcorners); CeedOperatorSetField(op_explicit, "v", Erestrictq, basisq, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_explicit, "dv", Erestrictq, basisq, @@ -419,25 +418,18 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedOperatorSetField(op_jacobian, "q", Erestrictq, basisq, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_jacobian, "deltaq", Erestrictq, basisq, - CEED_VECTOR_ACTIVE); // TO DO: Check restriction and basis for delta q + CEED_VECTOR_ACTIVE); // TODO: Check restriction and basis for delta q CeedOperatorSetField(op_jacobian, "qdata", Erestrictqdi, CEED_BASIS_COLLOCATED, qdata); CeedOperatorSetField(op_jacobian, "dv", Erestrictq, basisq, CEED_VECTOR_ACTIVE); user->op_jacobian = op_jacobian; - CeedQFunctionSetContext(qf_ics, &ctxSetup, sizeof ctxSetup); - // Set up the libCEED context - CeedScalar ctxSWICs[3] = {ctxSetup->u0, ctxSetup->v0, - ctxSetup->h0}; - CeedQFunctionSetContext(qf_ics, &ctxSWICs, sizeof ctxSWICs); - CeedScalar ctxSWExplicit = ctxSetup->f; - CeedQFunctionSetContext(qf_explicit, &ctxSWExplicit, sizeof ctxSWExplicit); - CeedScalar ctxSWImplicit = ctxSetup->g; - CeedQFunctionSetContext(qf_implicit, &ctxSWImplicit, sizeof ctxSWImplicit); - // same context for Jacobian qfunction - CeedQFunctionSetContext(qf_jacobian, &ctxSWImplicit, sizeof ctxSWImplicit); + CeedQFunctionSetContext(qf_ics, ctx, sizeof *ctx); + CeedQFunctionSetContext(qf_explicit, ctx, sizeof *ctx); + CeedQFunctionSetContext(qf_implicit, ctx, sizeof *ctx); + CeedQFunctionSetContext(qf_jacobian, ctx, sizeof *ctx); // Save libCEED data required for level // TODO: check how many of these are really needed outside data->basisx = basisx; diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index 4412dbcfd5..f2c6ec68b7 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -30,13 +30,21 @@ typedef struct { CeedScalar u0; CeedScalar v0; CeedScalar h0; - CeedScalar f; + CeedScalar Omega; + CeedScalar R; CeedScalar g; CeedScalar time; } PhysicsContext_s; typedef PhysicsContext_s *PhysicsContext; #endif // physics_context_struct +// MemType Options +static const char *const memTypes[] = { + "host", + "device", + "memType", "CEED_MEM_", NULL +}; + // Problem specific data typedef struct { CeedInt topodim, qdatasize; From 9bdada1e7fc7ad15380c0ac82b568051cc0300e6 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 30 Jun 2020 16:43:03 -0600 Subject: [PATCH 05/37] Update tap.sh after rebasing with master --- tests/tap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tap.sh b/tests/tap.sh index 400f2442a8..cf5d7cf5a9 100755 --- a/tests/tap.sh +++ b/tests/tap.sh @@ -31,9 +31,9 @@ elif [ ${1::4} == "nek-" ]; then for ((i=0;i<${numconfig};++i)); do allargs+=("$(awk -v i="$i" '/C_TESTARGS/,/\n/{j++}j==i+1{print; exit}' examples/nek/bps/${1:4}.usr* | cut -d\ -f2- )") done -elif [ ${1::7} == "fluids-navierstokes" ]; then +elif [ $1 == "fluids-navierstokes" ]; then # get all test configurations - numconfig=$(grep -F //TESTARGS examples/fluids/${1:7}.c* | wc -l) + numconfig=$(grep -F //TESTARGS examples/fluids/navier-stokes/${1:7}.c* | wc -l) for ((i=0;i<${numconfig};++i)); do allargs+=("$(awk -v i="$i" '/\/\/TESTARGS/,/\n/{j++}j==i+1{print; exit}' examples/fluids/navier-stokes/${1:7}.c | cut -d\ -f2- )") done From 1fb5d86d1fc87a12045ff94e8d701e34d3b8d1ef Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 30 Jun 2020 17:43:35 -0600 Subject: [PATCH 06/37] Add FormJacobian function --- examples/fluids/shallow-water/shallowwater.c | 17 +++--------- examples/fluids/shallow-water/src/wrappers.c | 29 ++++++++------------ 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 46a207ccbb..412b1b3d41 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -340,7 +340,7 @@ int main(int argc, char **argv) { ierr = ICs_FixMultiplicity(ceeddata->op_ics, ceeddata->xcorners, user->q0ceed, dm, Qloc, Q, ceeddata->Erestrictq, &ctx, 0.0); CHKERRQ(ierr); - + MPI_Comm_rank(comm, &rank); if (!rank) { ierr = PetscMkdir(user->outputfolder); CHKERRQ(ierr); @@ -364,25 +364,17 @@ int main(int argc, char **argv) { // Set up the MatShell for the associated Jacobian operator ierr = MatCreateShell(PETSC_COMM_SELF, 5*odofs, 5*odofs, PETSC_DETERMINE, PETSC_DETERMINE, user, &J); CHKERRQ(ierr); - // Set the MatShell user context -// ierr = MatShellSetContext(J, user); CHKERRQ(ierr); // Set the MatShell operation needed for the Jacobian ierr = MatShellSetOperation(J, MATOP_MULT, (void (*)(void))ApplyJacobian_SW); CHKERRQ(ierr); - // Set up the MatShell for the associated Jacobian preconditioning operator -// MatCreateShell(PETSC_COMM_SELF, lsize, lsize, PETSC_DETERMINE, -// PETSC_DETERMINE, (void*)&user, &Jpre); -// MatShellSetOperation(Jpre, MATOP_MATMAT_MULT, (void(*)(void))PreJacobianProductMat); - // Create and setup TS ierr = TSCreate(comm, &ts); CHKERRQ(ierr); ierr = TSSetDM(ts, dm); CHKERRQ(ierr); - ierr = TSSetType(ts,TSARKIMEX); CHKERRQ(ierr); - ierr = TSSetIFunction(ts, NULL, FormIFunction_SW, &user); CHKERRQ(ierr); + ierr = TSSetType(ts, TSARKIMEX); CHKERRQ(ierr); + // Tell the TS which functions to use for Explicit part (RHS), Implicit part and Jacobian ierr = TSSetRHSFunction(ts, NULL, FormRHSFunction_SW, &user); CHKERRQ(ierr); - // TODO: Check TSSetIJacobian - ierr = DMSetMatType(dm, MATSHELL); CHKERRQ(ierr); + ierr = TSSetIFunction(ts, NULL, FormIFunction_SW, &user); CHKERRQ(ierr); ierr = TSSetIJacobian(ts, J, J, FormJacobian_SW, &user); CHKERRQ(ierr); // Other TS options @@ -439,7 +431,6 @@ int main(int argc, char **argv) { steps,(double)ftime); CHKERRQ(ierr); } - // Clean up libCEED CeedVectorDestroy(&ceeddata->qdata); CeedVectorDestroy(&user->qceed); diff --git a/examples/fluids/shallow-water/src/wrappers.c b/examples/fluids/shallow-water/src/wrappers.c index 417841d546..5d4c0aa873 100644 --- a/examples/fluids/shallow-water/src/wrappers.c +++ b/examples/fluids/shallow-water/src/wrappers.c @@ -126,25 +126,20 @@ PetscErrorCode FormIFunction_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, PetscErrorCode FormJacobian_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, PetscReal sigma, Mat J, Mat Jpre, void *userData) { -// User user = *(User*)userData; -// PetscErrorCode ierr; + PetscErrorCode ierr; PetscFunctionBeginUser; - // empty body for now - -// // Set the preconditioning for the Jacobian, Jpre, to be = sigma M -// ierr = VecScale(user->M, sigma); CHKERRQ(ierr); -// ierr = VecCopy(M, JPreVec); CHKERRQ(ierr); - -// // Set up the MatShell for the associated Jacobian operator -// MatCreateShell(PETSC_COMM_SELF, lsize, lsize, PETSC_DETERMINE, -// PETSC_DETERMINE, (void*)&user, &J); -// MatShellSetOperation(J, MATOP_MULT, (void(*)(void))JacobianProductMat); - -// // Set up the MatShell for the associated Jacobian preconditioning operator -// MatCreateShell(PETSC_COMM_SELF, lsize, lsize, PETSC_DETERMINE, -// PETSC_DETERMINE, (void*)&user, &Jpre); -// MatShellSetOperation(Jpre, MATOP_MATMAT_MULT, (void(*)(void))PreJacobianProductMat); + + ierr = TSComputeIJacobianDefaultColor(ts, t, Q, Qdot, sigma, J, Jpre, NULL); + CHKERRQ(ierr); + + // Jpre might be AIJ (e.g., when using coloring), so we need to assemble it + ierr = MatAssemblyBegin(Jpre, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); + ierr = MatAssemblyEnd(Jpre, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); + if (J != Jpre) { + ierr = MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); + ierr = MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); + } PetscFunctionReturn(0); } From 05378a03679eeadc1521f3b6b3f2a98b361733a7 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 30 Jun 2020 22:45:41 -0600 Subject: [PATCH 07/37] Put H0 as a global variable and compute hs as part of qdata --- .../shallow-water/qfunctions/setup_geo.h | 3 + .../shallow-water/qfunctions/shallowwater.h | 84 +++++++++---------- examples/fluids/shallow-water/shallowwater.c | 21 +++-- examples/fluids/shallow-water/src/setup.c | 6 +- examples/fluids/shallow-water/sw_headers.h | 1 + 5 files changed, 57 insertions(+), 58 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/setup_geo.h b/examples/fluids/shallow-water/qfunctions/setup_geo.h index a36046bd6f..3949c10e50 100644 --- a/examples/fluids/shallow-water/qfunctions/setup_geo.h +++ b/examples/fluids/shallow-water/qfunctions/setup_geo.h @@ -171,6 +171,9 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, qdata[7][i] = dxdXTdxdXinv[0][0]; qdata[8][i] = dxdXTdxdXinv[1][1]; qdata[9][i] = dxdXTdxdXinv[0][1]; + + // Terrain topography, hs + qdata[10][i] = sin(xx[0]) + cos(xx[1]); // put 0 for constant flat topography } // End of Quadrature Point Loop // Return diff --git a/examples/fluids/shallow-water/qfunctions/shallowwater.h b/examples/fluids/shallow-water/qfunctions/shallowwater.h index 435f1952f2..65d7480f16 100644 --- a/examples/fluids/shallow-water/qfunctions/shallowwater.h +++ b/examples/fluids/shallow-water/qfunctions/shallowwater.h @@ -37,6 +37,7 @@ typedef struct { CeedScalar Omega; CeedScalar R; CeedScalar g; + CeedScalar H0; CeedScalar time; } PhysicsContext_s; typedef PhysicsContext_s *PhysicsContext; @@ -66,11 +67,6 @@ static inline int Exact_SW(CeedInt dim, CeedScalar time, const CeedScalar X[], q[0] = u0; q[1] = v0; q[2] = h0; - // Terrain topography, h_s - q[3] = sin(x) + cos(y); // put 0 for constant flat topography - // Reference height, H_0 - q[4] = 0; // flat - // Return return 0; } @@ -140,24 +136,23 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, for (CeedInt i=0; ig; + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar g = context->g; + const CeedScalar H0 = context->H0; CeedPragmaSIMD // Quadrature Point Loop for (CeedInt i=0; ig; + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar g = context->g; CeedPragmaSIMD // Quadrature Point Loop diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 412b1b3d41..057dc7fb19 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -61,18 +61,19 @@ int main(int argc, char **argv) { const CeedInt ncompx = 3; PetscInt viz_refine = 0; PetscBool read_mesh, simplex, test; - PetscInt topodim = 2, ncompq = 5, lnodes; + PetscInt topodim = 2, ncompq = 3, lnodes; // libCEED context char ceedresource[PETSC_MAX_PATH_LEN] = "/cpu/self", filename[PETSC_MAX_PATH_LEN]; Ceed ceed; CeedData ceeddata; CeedMemType memtyperequested; - PetscScalar meter = 1e-2; // 1 meter in scaled length units - PetscScalar second = 1e-2; // 1 second in scaled time units - PetscScalar Omega = 7.29212e-5; // Earth rotation rate (1/s) - PetscScalar g = 9.81; // gravitational acceleration (m/s^2) - PetscScalar R = 6.37122e6; // Earth radius (m) + PetscScalar meter = 1e-2; // 1 meter in scaled length units + PetscScalar second = 1e-2; // 1 second in scaled time units + PetscScalar Omega = 7.29212e-5; // Earth rotation rate (1/s) + PetscScalar R = 6.37122e6; // Earth radius (m) + PetscScalar g = 9.81; // gravitational acceleration (m/s^2) + PetscScalar H0 = 0; // constant mean height (m) PetscScalar mpersquareds; // Check PETSc CUDA support PetscBool petschavecuda, setmemtyperequest = PETSC_FALSE; @@ -127,6 +128,8 @@ int main(int argc, char **argv) { NULL, qextra, &qextra, NULL); CHKERRQ(ierr); ierr = PetscOptionsScalar("-g", "Gravitational acceleration", NULL, g, &g, NULL); CHKERRQ(ierr); + ierr = PetscOptionsScalar("-H0", "Mean height", + NULL, H0, &H0, NULL); CHKERRQ(ierr); PetscStrncpy(user->outputfolder, ".", 2); ierr = PetscOptionsString("-of", "Output folder", NULL, user->outputfolder, user->outputfolder, @@ -161,6 +164,7 @@ int main(int argc, char **argv) { .Omega = Omega, .R = R, .g = g, + .H0 = H0, .time = 0. }; @@ -362,8 +366,9 @@ int main(int argc, char **argv) { ierr = DMRestoreLocalVector(dm, &Qloc); CHKERRQ(ierr); // Set up the MatShell for the associated Jacobian operator - ierr = MatCreateShell(PETSC_COMM_SELF, 5*odofs, 5*odofs, PETSC_DETERMINE, - PETSC_DETERMINE, user, &J); CHKERRQ(ierr); + ierr = MatCreateShell(PETSC_COMM_SELF, ncompq*odofs, ncompq*odofs, + PETSC_DETERMINE, PETSC_DETERMINE, user, &J); + CHKERRQ(ierr); // Set the MatShell operation needed for the Jacobian ierr = MatShellSetOperation(J, MATOP_MULT, (void (*)(void))ApplyJacobian_SW); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index c9d39b0aa2..585bac2b19 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -136,10 +136,6 @@ PetscErrorCode SetupDM(DM dm, PetscInt degree, PetscInt ncompq, CHKERRQ(ierr); ierr = PetscSectionSetComponentName(section, 0, 2, "h"); CHKERRQ(ierr); - ierr = PetscSectionSetComponentName(section, 0, 3, "h_s"); - CHKERRQ(ierr); - ierr = PetscSectionSetComponentName(section, 0, 4, "H_0"); - CHKERRQ(ierr); } PetscFunctionReturn(0); } @@ -282,7 +278,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedOperator op_setup, op_ics, op_explicit, op_implicit, op_jacobian; CeedVector xcorners, qdata, q0ceed; - CeedInt P, Q, cStart, cEnd, nelem, qdatasize = 10; + CeedInt P, Q, cStart, cEnd, nelem, qdatasize = 11; // CEED bases P = degree + 1; diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index f2c6ec68b7..01e39c93f3 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -33,6 +33,7 @@ typedef struct { CeedScalar Omega; CeedScalar R; CeedScalar g; + CeedScalar H0; CeedScalar time; } PhysicsContext_s; typedef PhysicsContext_s *PhysicsContext; From fbd6f739854e78ee9827afc7ef3b80f08aa4ca89 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Thu, 2 Jul 2020 11:07:29 -0600 Subject: [PATCH 08/37] Add documentation --- doc/sphinx/source/references.bib | 28 +++++ examples/fluids/index.rst | 100 ++++++++++++++++++ .../shallow-water/qfunctions/shallowwater.h | 45 ++++---- 3 files changed, 155 insertions(+), 18 deletions(-) diff --git a/doc/sphinx/source/references.bib b/doc/sphinx/source/references.bib index ac2048b79f..225cda02b4 100644 --- a/doc/sphinx/source/references.bib +++ b/doc/sphinx/source/references.bib @@ -74,6 +74,15 @@ @article{papanastasiou1992outflow volume={14}, journal={International Journal for Numerical Methods in Fluids}, doi={10.1002/fld.1650140506} + +@article{rancic, + author = {Rančić, M. and Purser, R. J. and Mesinger, F.}, + title = {A global shallow-water model using an expanded spherical cube: Gnomonic versus conformal coordinates}, + journal = {Quarterly Journal of the Royal Meteorological Society}, + volume = {122}, + pages = {959-982}, + doi = {10.1002/qj.49712253209}, + year = {1996} } @article{straka1993numerical, @@ -88,6 +97,25 @@ @article{straka1993numerical doi={10.1002/fld.1650170103} } +@article{taylor2010ACA, + title={A compatible and conservative spectral element method on unstructured grids}, + author={Mark A. Taylor and Aim{\'e} Fournier}, + journal={J. Comput. Phys.}, + year={2010}, + volume={229}, + pages={5879-5895} +} + +@Article{ullrich, + author = {Ullrich, P. A.}, + title = {A global finite-element shallow-water model supporting continuous and discontinuous elements}, + journal = {Geoscientific Model Development}, + volume = {7}, + year = {2014}, + pages = {3017--3035}, + doi = {10.5194/gmd-7-3017-2014} +} + @article{williams2009roofline, title={Roofline: an insightful visual performance model for multicore architectures}, author={Williams, Samuel and Waterman, Andrew and Patterson, David}, diff --git a/examples/fluids/index.rst b/examples/fluids/index.rst index a346807b62..828f9b2333 100644 --- a/examples/fluids/index.rst +++ b/examples/fluids/index.rst @@ -341,3 +341,103 @@ and non-penetration boundary conditions for :math:`\bm{u}`, and no-flux for mass and energy densities. This problem can be run with:: ./navierstokes -problem density_current + + +.. _example-petsc-shallow-water: + +Shallow-water Equations mini-app +======================================== + +This example is located in the subdirectory :file:`examples/fluids/shallow-water`. It solves +the time-dependent shallow-water equations on a cubed-sphere. The geometry of the cubed-sphere and the relative coordinate transformations are explained in the section :ref:`example-petsc-area-sphere`. The discretization of most of the spatial differential terms needed in the shallow-water equations is already described in the section :ref:`example-petsc-bps-sphere`. + +The mathematical formulation (from :cite:`taylor2010ACA`) is given in what +follows + +.. math:: + :label: eq-swe + + \begin{aligned} + \frac{\partial \bm u}{\partial t} &= - (\omega + f) \bm{\hat k} \times \bm u - \nabla \left( \frac{|\bm u|^2}{2} + g (h + h_s) \right) \\ + \frac{\partial h}{\partial t} &= - \nabla \cdot (H_0 + h) \bm u \, , \\ + \end{aligned} + +where quantities are expressed in spherical coordinates :math:`r^1 = \lambda` for longitude, :math:`r^2 = \theta` for latitude and :math:`r^3 = r` for the radius, with associated unit vectors :math:`\bm{ \hat \lambda}`, :math:`\bm {\hat \theta}`, and :math:`\bm{\hat k}`, respectively. We consider a unit sphere, hence, :math:`r = 1` and :math:`\partial / \partial r = 0`. In equations :math:numref:`eq-swe`, :math:`\bm u` represents velocity field with longitudinal and latitudinal components :math:`\bm u = (u_{\lambda}, u_{\theta}) \equiv (u_1, u_2)`, and :math:`h` represents the height function of the fluid thickness; moreover, :math:`\omega = \nabla \times \bm u` is the vorticity, :math:`f` the Coriolis parameter such that :math:`f = 2 \bm{\Omega} \sin(\theta)` with :math:`\bm{\Omega}` Earth's rotation, :math:`g` represents the gravitational acceleration, :math:`h_s` the bottom surface (terrain) elevation, and :math:`H_0` the constant mean depth, such that the total fluid surface height is given by :math:`h_s + H_0 + h`. On the two-dimensional Riemannian manifold describing the sphere, we can use the coordinate indices :math:`x^s = {\alpha, \beta}`, so that we can express :math:`\bm u = u^{\alpha} \bm g_{\alpha} + u^{\beta} \bm g_{\beta}`, where :math:`\bm g_{\alpha} = \partial / \partial \alpha` and :math:`\bm g_{\beta} = \partial / \partial \beta` are the natural basis vectors on the manifold :cite:`ullrich,rancic`. + +We solve equations :math:numref:`eq-swe` for the state variable :math:`\bm q = (q_1, q_2, q_3) = (\bm u, h) = (u_{\lambda}, u_{\theta}, h) \equiv (u_1, u_2, h)` with a semi-implicit time integration, for which + +.. math:: + :label: eq-swe-semi-implicit + + \bm F(t, \bm q, \bm {\dot q}) = G(t, \bm q) \, , + +where the time derivative :math:`\bm{\dot q}` is defined by + +.. math:: + \bm{\dot q}(\bm q) = \sigma \bm q + \bm z + +in terms of :math:`\bm z` from prior state and :math:`\sigma > 0`, +both of which depend on the specific time integration scheme. We split the implicit and explicit terms as follows: + +.. math:: + :label: eq-swe-implicit-part + + \bm F(t, \bm q, \bm {\dot q}) := + \left\{ + \begin{array}{l} + F_1 (u_{\lambda}, u_{\theta}, h) = \frac{\partial u_{\lambda}}{\partial t} + g \frac{\partial }{\partial \alpha} \left( h + h_s \right)\\ + F_2 (u_{\lambda}, u_{\theta}, h) = \frac{\partial u_{\theta}}{\partial t} + g \frac{\partial }{\partial \beta} \left( h + h_s \right)\\ + F_3 (u_{\lambda}, u_{\theta}, h) = \frac{\partial h}{\partial t} + \frac{\partial }{\partial \alpha} \left( (H_0 + h) u_{\lambda} \right) + \frac{\partial }{\partial \beta} \left( (H_0 + h) u_{\theta} \right) . + \end{array} + \right. + +While for the explicit part we specify + +.. math:: + :label: eq-swe-explicit-part + + \bm G(t, \bm q) := + \left\{ + \begin{array}{l} + G_1 (u_{\lambda}, u_{\theta}, h) = - u_{\lambda} \frac{\partial u_{\lambda}}{\partial \alpha} - u_{\theta} \frac{\partial u_{\lambda}}{\partial \beta} - f u_{\theta}\\ + G_2 (u_{\lambda}, u_{\theta}, h) = - u_{\lambda} \frac{\partial u_{\theta}}{\partial \alpha} - u_{\theta} \frac{\partial u_{\theta}}{\partial \beta} + f u_{\lambda}\\ + G_3 (u_{\lambda}, u_{\theta}, h) = 0 . + \end{array} + \right. + +The differentiation of the implicit part :math:numref:`eq-swe-implicit-part` with respect to the increment :math:`\delta \bm q = (\delta \bm u, \delta h)` gives the strong form of the incremental Jacobian :math:`\partial F_i / \partial \delta q_j`, with :math:`i,j = 1,2,3` + +.. math:: + :label: eq-strong-swe-Jacobian + + \frac{\partial \bm F}{\partial \delta \bm q} := + \left( + \begin{array}{ccc} + \frac{\partial F_1}{\partial \delta u_{\lambda}} & \frac{\partial F_1}{\partial \delta u_{\theta}} & \frac{\partial F_1}{\partial \delta h}\\ + \frac{\partial F_2}{\partial \delta u_{\lambda}} & \frac{\partial F_2}{\partial \delta u_{\theta}} & \frac{\partial F_2}{\partial \delta h}\\ + \frac{\partial F_3}{\partial \delta u_{\lambda}} & \frac{\partial F_3}{\partial \delta u_{\theta}} & \frac{\partial F_3}{\partial \delta h} + \end{array} + \right) + = + \left( + \begin{array}{ccc} + 0 & 0 & g \partial (\delta h) / \partial \alpha \\ + 0 & 0 & g \partial (\delta h) / \partial \beta \\ + \frac{\partial ((H_0 + h) \delta u_{\lambda})}{\partial \alpha} & \frac{\partial ((H_0 + h) \delta u_{\theta})}{\partial \beta} & \frac{\partial (\delta h u_{\lambda})}{\partial \alpha} + \frac{\partial (\delta h u_{\theta})}{\partial \beta} , + \end{array} + \right) + +whose integrands of the weak form are found by multiplying each entry of :math:`\partial F_i / \partial \delta q_j` by a test function :math:`\phi \in H^1(\Omega)` +and integrating by parts + +.. math:: + :label: eq-weak-swe-Jacobian + + \left( + \begin{array}{ccc} + 0 & 0 & - g \delta h \partial \phi / \partial \alpha \\ + 0 & 0 & - g \delta h \partial \phi / \partial \beta \\ + - \frac{\partial \phi}{\partial \alpha} ((H_0 + h) \delta u_{\lambda}) & - \frac{\partial \phi}{\partial \beta} ((H_0 + h) \delta u_{\theta}) & - \frac{\partial \phi}{\partial \alpha} (\delta h u_{\lambda}) - \frac{\partial \phi}{\partial \beta} (\delta h u_{\theta}) + \end{array} + \right) . + diff --git a/examples/fluids/shallow-water/qfunctions/shallowwater.h b/examples/fluids/shallow-water/qfunctions/shallowwater.h index 65d7480f16..91acbfbaea 100644 --- a/examples/fluids/shallow-water/qfunctions/shallowwater.h +++ b/examples/fluids/shallow-water/qfunctions/shallowwater.h @@ -104,14 +104,16 @@ CEED_QFUNCTION(ICsSW)(void *ctx, CeedInt Q, // equations // // The equations represent 2D shallow-water flow on a spherical surface, where -// the state variables, u_lambda, u_theta, represent the longitudinal and latitudinal components -// of the velocity field, and h, represents the height function. +// the state variables, u_lambda, u_theta (or u_1, u_2) represent the +// longitudinal and latitudinal components of the velocity field, and h, +// represents the height function. // -// State variable vector: q = (u_lambda,u_theta,h) +// State variable vector: q = (u_lambda, u_theta, h) // -// Shallow-water Equations spatial terms of explicit function G(t,q) = (G_1(t,q), G_2(t,q)): -// G_1(t,q) = - (omega + f) * khat curl u - grad(|u|^2/2) -// G_2(t,q) = 0 +// Shallow-water Equations spatial terms of explicit function +// G(t,q) = (G_1(t,q), G_2(t,q)): +// G_1(t,q) = - (omega + f) * khat curl u - grad(|u|^2/2) +// G_2(t,q) = 0 // ***************************************************************************** CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { @@ -145,10 +147,10 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, q[1][i] }; // *INDENT-OFF* - const CeedScalar du[2][2] = {{dq[0][0][i], // du/dx - dq[1][0][i]}, // du/dy - {dq[0][1][i], // dv/dx - dq[1][1][i]} // dv/dy + const CeedScalar du[2][2] = {{dq[0][0][i], // du_1/dx + dq[1][0][i]}, // du_1/dy + {dq[0][1][i], // du_2/dx + dq[1][1][i]} // du_2/dy }; // *INDENT-ON* // Interp-to-Interp qdata @@ -189,14 +191,18 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // equations // // The equations represent 2D shallow-water flow on a spherical surface, where -// the state variables, u_lambda, u_theta, represent the longitudinal and latitudinal components -// of the velocity field, and h, represents the height function. +// the state variables, u_lambda, u_theta (or u_1, u_2) represent the +// longitudinal and latitudinal components of the velocity field, and h, +// represents the height function. // // State variable vector: q = (u_lambda, u_theta, h) // -// Shallow-water Equations spatial terms of implicit function: F(t,q) = (F_1(t,q), F_2(t,q)): -// F_1(t,q) = g(grad(h + h_s)) -// F_2(t,q) = div((h + H_0) u) +// Shallow-water Equations spatial terms of implicit function: +// F(t,q) = (F_1(t,q), F_2(t,q)): +// F_1(t,q) = g(grad(h + h_s)) +// F_2(t,q) = div((h + H_0) u) +// +// To the spatial term F(t,q) one needs to add qdot (time derivative) on the LHS // ***************************************************************************** CEED_QFUNCTION(SWImplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { @@ -280,10 +286,13 @@ CEED_QFUNCTION(SWImplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // equations // // The equations represent 2D shallow-water flow on a spherical surface, where -// the state variables, u_lambda, u_theta, represent the longitudinal and latitudinal components -// of the velocity field, and h, represents the height function. +// the state variables, u_lambda, u_theta (or u_1, u_2) represent the +// longitudinal and latitudinal components of the velocity field, and h, +// represents the height function. // -// Discrete Jacobian: dF/dq^n = sigma * dF/dqdot|q^n + dF/dq|q^n ("sigma * dF/dqdot|q^n" will be added later) +// Discrete Jacobian: +// dF/dq^n = sigma * dF/dqdot|q^n + dF/dq|q^n +// ("sigma * dF/dqdot|q^n" will be added later) // ***************************************************************************** CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { From 48f757145f77edcf34694ae5c27deb5887e28e67 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 6 Jul 2020 18:35:30 -0600 Subject: [PATCH 09/37] SWE: Add advection test case and skeleton for geostrophic test --- .../shallow-water/qfunctions/advection.h | 433 ++++++++++++++++++ .../{shallowwater.h => geostrophic.h} | 75 ++- examples/fluids/shallow-water/shallowwater.c | 75 ++- examples/fluids/shallow-water/src/setup.c | 73 ++- examples/fluids/shallow-water/sw_headers.h | 72 ++- 5 files changed, 637 insertions(+), 91 deletions(-) create mode 100644 examples/fluids/shallow-water/qfunctions/advection.h rename examples/fluids/shallow-water/qfunctions/{shallowwater.h => geostrophic.h} (87%) diff --git a/examples/fluids/shallow-water/qfunctions/advection.h b/examples/fluids/shallow-water/qfunctions/advection.h new file mode 100644 index 0000000000..59bdb26480 --- /dev/null +++ b/examples/fluids/shallow-water/qfunctions/advection.h @@ -0,0 +1,433 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Initial condition and operator for the shallow-water example using PETSc + +#ifndef advection_h +#define advection_h + +#include "../sw_headers.h" + +#ifndef __CUDACC__ +# include +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +// ***************************************************************************** +// This QFunction sets the the initial condition for the advection of a cosine +// bell shaped scalar function (test case 1 in "A Standard Test Set for +// Numerical Approximations to the Shallow Water Equations in Spherical +// Geometry" by Williamson et al. (1992) +// ***************************************************************************** +static inline int Exact_SW_Advection(CeedInt dim, CeedScalar time, + const CeedScalar X[], CeedInt Nf, + CeedScalar q[], void *ctx) { + + // Context + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar u0 = context->u0; + const CeedScalar h0 = context->h0; + const CeedScalar R = context->R; + const CeedScalar gamma = context->gamma; + const CeedScalar rho = R / 3.; + + // Setup + // -- Compute latitude + const CeedScalar theta = asin(X[2] / R); + // -- Compute longitude + const CeedScalar lambda = atan2(X[1], X[0]); + // -- Compute great circle distance between (lambda, theta) and the center, + // (lambda_c, theta_c) + const CeedScalar lambda_c = 3. * M_PI / 2.; + const CeedScalar theta_c = 0.; + const CeedScalar r = R * acos(sin(theta_c)*sin(theta) + cos(theta_c)*cos(theta)*cos(lambda-lambda_c)); + + // Initial Conditions + q[0] = u0 * (cos(theta)*cos(gamma) + sin(theta)*cos(lambda)*sin(gamma)); + q[1] = -u0 * sin(lambda)*sin(gamma); + q[2] = r < rho ? .5*h0*(1 + cos(M_PI*r/rho)) : 0.; // cosine bell + // Return + return 0; +} + +// ***************************************************************************** +// Initial conditions +// ***************************************************************************** +CEED_QFUNCTION(ICsSW_Advection)(void *ctx, CeedInt Q, + const CeedScalar *const *in, CeedScalar *const *out) { + // Inputs + const CeedScalar (*X)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0]; + + // Outputs + CeedScalar (*q0)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; iH0; + const CeedScalar CtauS = context->CtauS; + const CeedScalar strong_form = context->strong_form; + + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; iH0; + const CeedScalar CtauS = context->CtauS; + const CeedScalar strong_form = context->strong_form; + + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; istabilization) { + case 0: + break; + case 1: dv[j][2][i] += wdetJ * TauS * strongConv * uX[j]; //SU + break; + case 2: dv[j][2][i] += wdetJ * TauS * strongResid * uX[j]; //SUPG + break; + } + } // End Quadrature Point Loop + + return 0; + } + +// ***************************************************************************** +// This QFunction implements the Jacobian of of the advection test for the +// shallow-water equations solver. +// +// For this simple test, the equation represents the tranport of the scalar +// field h advected by the wind u, on a spherical surface, where the state +// variables, u_lambda, u_theta (or u_1, u_2) represent the longitudinal +// and latitudinal components of the velocity (wind) field, and h, represents +// the height function. +// +// Discrete Jacobian: +// dF/dq^n = sigma * dF/dqdot|q^n + dF/dq|q^n +// ("sigma * dF/dqdot|q^n" will be added later) +// ***************************************************************************** +CEED_QFUNCTION(SWJacobian_Advection)(void *ctx, CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*q)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], + (*deltaq)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + // Outputs + CeedScalar (*deltadvdX)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[1]; + // *INDENT-ON* + // Context + ProblemContext context = (ProblemContext)ctx; + const CeedScalar H0 = context->H0; + + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; i @@ -28,51 +30,44 @@ #define M_PI 3.14159265358979323846 #endif -#ifndef physics_context_struct -#define physics_context_struct -typedef struct { - CeedScalar u0; - CeedScalar v0; - CeedScalar h0; - CeedScalar Omega; - CeedScalar R; - CeedScalar g; - CeedScalar H0; - CeedScalar time; -} PhysicsContext_s; -typedef PhysicsContext_s *PhysicsContext; -#endif // physics_context_struct - // ***************************************************************************** -// This QFunction sets the the initial conditions and boundary conditions -// -// For now we have sinusoidal terrain and constant reference height H0 -// +// This QFunction sets the the initial condition for the steady state nonlinear +// zonal geostrophic flow (test case 2 in "A Standard Test Set for Numerical +// Approximations to the Shallow Water Equations in Spherical Geometry" +// by Williamson et al. (1992) // ***************************************************************************** -static inline int Exact_SW(CeedInt dim, CeedScalar time, const CeedScalar X[], +static inline int Exact_SW(CeedInt dim, CeedScalar time, const CeedScalar X[], CeedInt Nf, CeedScalar q[], void *ctx) { // Context const PhysicsContext context = (PhysicsContext)ctx; const CeedScalar u0 = context->u0; - const CeedScalar v0 = context->v0; const CeedScalar h0 = context->h0; + const CeedScalar R = context->R; + const CeedScalar gamma = context->gamma; + const CeedScalar rho = R / 3.; // Setup - // -- Coordinates - const CeedScalar x = X[0]; - const CeedScalar y = X[1]; + // -- Compute latitude + const CeedScalar theta = asin(X[2] / R); + // -- Compute longitude + const CeedScalar lambda = atan2(X[1], X[0]); + // -- Compute great circle distance between (lambda, theta) and the center, + // (lambda_c, theta_c) + const CeedScalar lambda_c = 3. * M_PI / 2.; + const CeedScalar theta_c = 0.; + const CeedScalar r = R * acos(sin(theta_c)*sin(theta) + cos(theta_c)*cos(theta)*cos(lambda-lambda_c)); // Initial Conditions - q[0] = u0; - q[1] = v0; - q[2] = h0; + q[0] = u0 * (cos(theta)*cos(gamma) + sin(theta)*cos(lambda)*sin(gamma)); + q[1] = -u0 * sin(lambda)*sin(gamma); + q[2] = r < rho ? .5*h0*(1 + cos(M_PI*r/rho)) : 0.; // cosine bell // Return return 0; } // ***************************************************************************** -// Initial conditions for shallow-water +// Initial conditions // ***************************************************************************** CEED_QFUNCTION(ICsSW)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { @@ -98,7 +93,6 @@ CEED_QFUNCTION(ICsSW)(void *ctx, CeedInt Q, return 0; } - // ***************************************************************************** // This QFunction implements the explicit terms of the shallow-water // equations @@ -121,11 +115,11 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Inputs const CeedScalar (*X)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], (*q)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], - (*dq)[5][CEED_Q_VLA] = (const CeedScalar(*)[5][CEED_Q_VLA])in[1], + (*dq)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], - (*dv)[5][CEED_Q_VLA] = (CeedScalar(*)[5][CEED_Q_VLA])out[1]; + (*dv)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[1]; // *INDENT-ON* // Context @@ -209,8 +203,8 @@ CEED_QFUNCTION(SWImplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // *INDENT-OFF* // Inputs const CeedScalar (*q)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], - (*qdot)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], - (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + (*qdot)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[3]; // Outputs CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], (*dv)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[1]; @@ -299,7 +293,7 @@ CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, // *INDENT-OFF* // Inputs const CeedScalar (*q)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], - (*deltaq)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*deltaq)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs CeedScalar (*deltadvdX)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[1]; @@ -307,6 +301,7 @@ CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Context const PhysicsContext context = (PhysicsContext)ctx; const CeedScalar g = context->g; + const CeedScalar H0 = context->H0; CeedPragmaSIMD // Quadrature Point Loop @@ -314,8 +309,6 @@ CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Setup // Interp in const CeedScalar h = q[2][i]; - // H0 - const CeedScalar H_0 = q[4][i]; // Functional derivatives in const CeedScalar deltau[2] = {deltaq[0][0][i], deltaq[1][0][i] @@ -347,8 +340,8 @@ CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, deltadvdX[1][1][i] = - g*wdetJ*dXdx[1][1]*deltah; // theta component // Jacobian spatial terms for F_3(t,q): // - dv \cdot ((H_0 + h) delta u) - deltadvdX[1][2][i] = - (H_0 + h)*wdetJ*(deltau[0]*dXdx[1][0] + deltau[1]*dXdx[1][1]); // theta component - deltadvdX[0][2][i] = - (H_0 + h)*wdetJ*(deltau[0]*dXdx[0][0] + deltau[1]*dXdx[0][1]); // lambda component + deltadvdX[1][2][i] = - (H0 + h)*wdetJ*(deltau[0]*dXdx[1][0] + deltau[1]*dXdx[1][1]); // lambda component + deltadvdX[0][2][i] = - (H0 + h)*wdetJ*(deltau[0]*dXdx[0][0] + deltau[1]*dXdx[0][1]); // theta component } // End Quadrature Point Loop @@ -358,4 +351,4 @@ CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, // ***************************************************************************** -#endif +#endif // shallowwater_h diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 057dc7fb19..4853e725f8 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -54,6 +54,10 @@ int main(int argc, char **argv) { Mat J, interpviz; User user; Units units; + problemType problemChoice; + problemData *problem = NULL; + StabilizationType stab; + PetscBool implicit; PetscInt degree, qextra, outputfreq, steps, contsteps; PetscMPIInt rank; PetscScalar ftime; @@ -68,12 +72,15 @@ int main(int argc, char **argv) { Ceed ceed; CeedData ceeddata; CeedMemType memtyperequested; - PetscScalar meter = 1e-2; // 1 meter in scaled length units - PetscScalar second = 1e-2; // 1 second in scaled time units - PetscScalar Omega = 7.29212e-5; // Earth rotation rate (1/s) - PetscScalar R = 6.37122e6; // Earth radius (m) - PetscScalar g = 9.81; // gravitational acceleration (m/s^2) - PetscScalar H0 = 0; // constant mean height (m) + CeedScalar CtauS = 0.; // dimensionless + CeedScalar strong_form = 0.; // [0,1] + PetscScalar meter = 1e-2; // 1 meter in scaled length units + PetscScalar second = 1e-2; // 1 second in scaled time units + PetscScalar Omega = 7.29212e-5; // Earth rotation rate (1/s) + PetscScalar R = 6.37122e6; // Earth radius (m) + PetscScalar g = 9.81; // gravitational acceleration (m/s^2) + PetscScalar H0 = 0; // constant mean height (m) + PetscScalar gamma = 0; // angle between axis of rotation and polar axis PetscScalar mpersquareds; // Check PETSc CUDA support PetscBool petschavecuda, setmemtyperequest = PETSC_FALSE; @@ -104,6 +111,38 @@ int main(int argc, char **argv) { sizeof(ceedresource), NULL); CHKERRQ(ierr); ierr = PetscOptionsBool("-test", "Run in test mode", NULL, test=PETSC_FALSE, &test, NULL); CHKERRQ(ierr); + problemChoice = SWE_ADVECTION; + ierr = PetscOptionsEnum("-problem", "Problem to solve", NULL, + problemTypes, (PetscEnum)problemChoice, + (PetscEnum *)&problemChoice, NULL); CHKERRQ(ierr); + problem = &problemOptions[problemChoice]; + ierr = PetscOptionsEnum("-stab", "Stabilization method", NULL, + StabilizationTypes, (PetscEnum)(stab = STAB_NONE), + (PetscEnum *)&stab, NULL); CHKERRQ(ierr); + ierr = PetscOptionsBool("-implicit", "Use implicit (IFunction) formulation", + NULL, implicit=PETSC_FALSE, &implicit, NULL); + CHKERRQ(ierr); + if (!implicit && stab != STAB_NONE) { + ierr = PetscPrintf(comm, "Warning! Use -stab only with -implicit\n"); + CHKERRQ(ierr); + } + ierr = PetscOptionsScalar("-CtauS", + "Scale coefficient for tau (nondimensional)", + NULL, CtauS, &CtauS, NULL); CHKERRQ(ierr); + if (stab == STAB_NONE && CtauS != 0) { + ierr = PetscPrintf(comm, + "Warning! Use -CtauS only with -stab su or -stab supg\n"); + CHKERRQ(ierr); + } + ierr = PetscOptionsScalar("-strong_form", + "Strong (1) or weak/integrated by parts (0) advection residual", + NULL, strong_form, &strong_form, NULL); + CHKERRQ(ierr); + if (problemChoice == SWE_GEOSTROPHIC && (CtauS != 0 || strong_form != 0)) { + ierr = PetscPrintf(comm, + "Warning! Problem geostrophic does not support -CtauS or -strong_form\n"); + CHKERRQ(ierr); + } ierr = PetscOptionsInt("-viz_refine", "Regular refinement levels for visualization", NULL, viz_refine, &viz_refine, NULL); @@ -130,6 +169,8 @@ int main(int argc, char **argv) { NULL, g, &g, NULL); CHKERRQ(ierr); ierr = PetscOptionsScalar("-H0", "Mean height", NULL, H0, &H0, NULL); CHKERRQ(ierr); + ierr = PetscOptionsScalar("-H0", "Angle between axis of rotation and polar axis", + NULL, gamma, &gamma, NULL); CHKERRQ(ierr); PetscStrncpy(user->outputfolder, ".", 2); ierr = PetscOptionsString("-of", "Output folder", NULL, user->outputfolder, user->outputfolder, @@ -157,7 +198,7 @@ int main(int argc, char **argv) { g *= mpersquareds; // Set up the libCEED context - PhysicsContext_s ctx = { + PhysicsContext_s phys_ctx = { .u0 = 0., .v0 = 0., .h0 = .1, @@ -165,8 +206,16 @@ int main(int argc, char **argv) { .R = R, .g = g, .H0 = H0, + .gamma = gamma, .time = 0. }; + + ProblemContext_s probl_ctx = { + .H0 = H0, + .CtauS = CtauS, + .strong_form = strong_form, + .stabilization = stab + }; // Setup DM if (read_mesh) { @@ -277,6 +326,9 @@ int main(int argc, char **argv) { ierr = PetscPrintf(comm, "\n-- CEED Shallow-water equations solver on the cubed-sphere -- libCEED + PETSc --\n" " rank(s) : %d\n" + " Problem:\n" + " Problem Name : %s\n" + " Stabilization : %s\n" " PETSc:\n" " PETSc Vec Type : %s\n" " libCEED:\n" @@ -292,7 +344,8 @@ int main(int argc, char **argv) { " DoFs per node : %D\n" " Global DoFs : %D\n" " Owned DoFs : %D\n", - comm_size, ceedresource, usedresource, + comm_size, problemTypes[problemChoice], + StabilizationTypes[stab], ceedresource, usedresource, CeedMemTypes[memtypebackend], (setmemtyperequest) ? CeedMemTypes[memtyperequested] : "none", numP, numQ, @@ -307,8 +360,8 @@ int main(int argc, char **argv) { // Setup libCEED's objects ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); - ierr = SetupLibceed(dm, ceed, degree, topodim, qextra, - ncompx, ncompq, user, ceeddata, &ctx); CHKERRQ(ierr); + ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, ceeddata, + problem, &phys_ctx, &probl_ctx); CHKERRQ(ierr); // Set up PETSc context // Set up units structure @@ -343,7 +396,7 @@ int main(int argc, char **argv) { // Apply IC operator and fix multiplicity of initial state vector ierr = ICs_FixMultiplicity(ceeddata->op_ics, ceeddata->xcorners, user->q0ceed, dm, Qloc, Q, ceeddata->Erestrictq, - &ctx, 0.0); CHKERRQ(ierr); + &phys_ctx, 0.0); CHKERRQ(ierr); MPI_Comm_rank(comm, &rank); if (!rank) { diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 585bac2b19..03572295c8 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -25,13 +25,47 @@ #include #include "../sw_headers.h" // Function prototytes #include "../qfunctions/setup_geo.h" // Geometric factors -#include "../qfunctions/shallowwater.h" // Physics point-wise functions +#include "../qfunctions/advection.h" // Physics point-wise functions +#include "../qfunctions/geostrophic.h" // Physics point-wise functions #if PETSC_VERSION_LT(3,14,0) # define DMPlexGetClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexGetClosureIndices(a,b,c,d,f,g,i) # define DMPlexRestoreClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexRestoreClosureIndices(a,b,c,d,f,g,i) #endif +problemData problemOptions[] = { + [SWE_ADVECTION] = { + .topodim = 2, + .qdatasize = 11, + .setup = SetupGeo, + .setup_loc = SetupGeo_loc, + .ics = ICsSW_Advection, + .ics_loc = ICsSW_Advection_loc, + .apply_explfunction = SWExplicit_Advection, + .apply_explfunction_loc = SWExplicit_Advection_loc, + .apply_implfunction = SWImplicit_Advection, + .apply_implfunction_loc = SWImplicit_Advection_loc, + .apply_jacobian = SWJacobian_Advection, + .apply_jacobian_loc = SWJacobian_Advection_loc, + .non_zero_time = PETSC_TRUE + }, + [SWE_GEOSTROPHIC] = { + .topodim = 2, + .qdatasize = 11, + .setup = SetupGeo, + .setup_loc = SetupGeo_loc, + .ics = ICsSW, + .ics_loc = ICsSW_loc, + .apply_explfunction = SWExplicit, + .apply_explfunction_loc = SWExplicit_loc, + .apply_implfunction = SWImplicit, + .apply_implfunction_loc = SWImplicit_loc, + .apply_jacobian = SWJacobian, + .apply_jacobian_loc = SWJacobian_loc, + .non_zero_time = PETSC_FALSE + } +}; + // ----------------------------------------------------------------------------- // Auxiliary function to create PETSc FE space for a given degree // ----------------------------------------------------------------------------- @@ -263,11 +297,10 @@ PetscErrorCode VectorPlacePetscVec(CeedVector c, Vec p) { // Auxiliary function to set up libCEED objects for a given degree // ----------------------------------------------------------------------------- -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, - CeedInt topodim, CeedInt qextra, - PetscInt ncompx, PetscInt ncompq, - User user, CeedData data, - PhysicsContext ctx) { +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, + PetscInt ncompx, PetscInt ncompq, User user, + CeedData data, problemData *problem, + PhysicsContext phys_ctx, ProblemContext probl_ctx) { int ierr; DM dmcoord; Vec Xloc; @@ -278,7 +311,8 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedOperator op_setup, op_ics, op_explicit, op_implicit, op_jacobian; CeedVector xcorners, qdata, q0ceed; - CeedInt P, Q, cStart, cEnd, nelem, qdatasize = 11; + CeedInt P, Q, cStart, cEnd, nelem, qdatasize = problem->qdatasize, + topodim = problem->topodim;; // CEED bases P = degree + 1; @@ -317,20 +351,21 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, user->q0ceed = q0ceed; // Create the Q-Function that builds the quadrature data - CeedQFunctionCreateInterior(ceed, 1, SetupGeo, SetupGeo_loc, &qf_setup); + CeedQFunctionCreateInterior(ceed, 1, problem->setup, problem->setup_loc, + &qf_setup); CeedQFunctionAddInput(qf_setup, "x", ncompx, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_setup, "dx", ncompx*topodim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_setup, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddOutput(qf_setup, "qdata", qdatasize, CEED_EVAL_NONE); // Create the Q-Function that sets the ICs of the operator - CeedQFunctionCreateInterior(ceed, 1, ICsSW, ICsSW_loc, &qf_ics); + CeedQFunctionCreateInterior(ceed, 1, problem->ics, problem->ics_loc, &qf_ics); CeedQFunctionAddInput(qf_ics, "x", ncompx, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_ics, "q0", ncompq, CEED_EVAL_NONE); // Create the Q-Function that defines the explicit part of the PDE operator - CeedQFunctionCreateInterior(ceed, 1, SWExplicit, SWExplicit_loc, - &qf_explicit); + CeedQFunctionCreateInterior(ceed, 1, problem->apply_explfunction, + problem->apply_explfunction_loc, &qf_explicit); CeedQFunctionAddInput(qf_explicit, "x", ncompx, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_explicit, "q", ncompq, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_explicit, "dq", ncompq*topodim, CEED_EVAL_GRAD); @@ -339,8 +374,8 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedQFunctionAddOutput(qf_explicit, "dv", ncompq*topodim, CEED_EVAL_GRAD); // Create the Q-Function that defines the implicit part of the PDE operator - CeedQFunctionCreateInterior(ceed, 1, SWImplicit, SWImplicit_loc, - &qf_implicit); + CeedQFunctionCreateInterior(ceed, 1, problem->apply_implfunction, + problem->apply_implfunction_loc, &qf_implicit); CeedQFunctionAddInput(qf_implicit, "q", ncompq, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_implicit, "dq", ncompq*topodim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_implicit, "qdot", ncompq, CEED_EVAL_INTERP); @@ -350,8 +385,8 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedQFunctionAddOutput(qf_implicit, "dv", ncompq*topodim, CEED_EVAL_GRAD); // Create the Q-Function that defines the action of the Jacobian operator - CeedQFunctionCreateInterior(ceed, 1, SWJacobian, SWJacobian_loc, - &qf_jacobian); + CeedQFunctionCreateInterior(ceed, 1, problem->apply_jacobian, + problem->apply_jacobian_loc, &qf_jacobian); CeedQFunctionAddInput(qf_jacobian, "q", 3, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_jacobian, "deltaq", 3, CEED_EVAL_NONE); CeedQFunctionAddInput(qf_jacobian, "qdata", 10, CEED_EVAL_NONE); @@ -422,10 +457,10 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, user->op_jacobian = op_jacobian; // Set up the libCEED context - CeedQFunctionSetContext(qf_ics, ctx, sizeof *ctx); - CeedQFunctionSetContext(qf_explicit, ctx, sizeof *ctx); - CeedQFunctionSetContext(qf_implicit, ctx, sizeof *ctx); - CeedQFunctionSetContext(qf_jacobian, ctx, sizeof *ctx); + CeedQFunctionSetContext(qf_ics, phys_ctx, sizeof *phys_ctx); + CeedQFunctionSetContext(qf_explicit, probl_ctx, sizeof *probl_ctx); + CeedQFunctionSetContext(qf_implicit, probl_ctx, sizeof *probl_ctx); + CeedQFunctionSetContext(qf_jacobian, probl_ctx, sizeof *probl_ctx); // Save libCEED data required for level // TODO: check how many of these are really needed outside data->basisx = basisx; diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index 01e39c93f3..6b216b5c88 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -24,8 +24,10 @@ #include #include -#ifndef physics_context_struct -#define physics_context_struct +// ----------------------------------------------------------------------------- +// Data Structs +// ----------------------------------------------------------------------------- + typedef struct { CeedScalar u0; CeedScalar v0; @@ -35,26 +37,58 @@ typedef struct { CeedScalar g; CeedScalar H0; CeedScalar time; + CeedScalar gamma; } PhysicsContext_s; typedef PhysicsContext_s *PhysicsContext; -#endif // physics_context_struct -// MemType Options -static const char *const memTypes[] = { - "host", - "device", - "memType", "CEED_MEM_", NULL -}; +typedef struct { + CeedScalar H0; + CeedScalar CtauS; + CeedScalar strong_form; + int stabilization; // See StabilizationType: 0=none, 1=SU, 2=SUPG +} ProblemContext_s; +typedef ProblemContext_s *ProblemContext; // Problem specific data typedef struct { CeedInt topodim, qdatasize; - CeedQFunctionUser setup, ics, apply_explfunction, apply_implfunction; + CeedQFunctionUser setup, ics, apply_explfunction, apply_implfunction, + apply_jacobian; const char *setup_loc, *ics_loc, *apply_explfunction_loc, - *apply_implfunction_loc; + *apply_implfunction_loc, *apply_jacobian_loc; const bool non_zero_time; } problemData; +// MemType Options +static const char *const memTypes[] = { + "host", + "device", + "memType", "CEED_MEM_", NULL +}; + +// Problem Options +typedef enum { + SWE_ADVECTION = 0, + SWE_GEOSTROPHIC = 1 +} problemType; +static const char *const problemTypes[] = { + "advection", + "geostrophic", + "problemType", "SWE_", NULL +}; + +typedef enum { + STAB_NONE = 0, + STAB_SU = 1, // Streamline Upwind + STAB_SUPG = 2, // Streamline Upwind Petrov-Galerkin +} StabilizationType; +static const char *const StabilizationTypes[] = { + "none", + "SU", + "SUPG", + "StabilizationType", "STAB_", NULL +}; + // PETSc user data typedef struct User_ *User; typedef struct Units_ *Units; @@ -82,10 +116,6 @@ struct Units_ { PetscScalar mpersquareds; }; -// ----------------------------------------------------------------------------- -// libCEED Data Struct -// ----------------------------------------------------------------------------- - // libCEED data struct typedef struct CeedData_ *CeedData; struct CeedData_ { @@ -98,6 +128,9 @@ struct CeedData_ { CeedVector xcorners, xceed, qdata, q0ceed, mceed, hsceed, H0ceed; }; +// External variables +extern problemData problemOptions[]; + // ----------------------------------------------------------------------------- // Setup DM functions // ----------------------------------------------------------------------------- @@ -119,11 +152,10 @@ PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, CeedInt ncomp, CeedElemRestriction *Erestrict); // Auxiliary function to set up libCEED objects for a given degree -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, - CeedInt topodim, CeedInt qextra, - PetscInt ncompx, PetscInt ncompq, - User user, CeedData data, - PhysicsContext ctx); +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, + PetscInt ncompx, PetscInt ncompq, User user, + CeedData data, problemData *problem, + PhysicsContext phys_ctx, ProblemContext probl_ctx); // ----------------------------------------------------------------------------- // RHS (Explicit part in time-stepper) function setup From d909027b118d057ddc8fd87b2a0df94d36f4cb5f Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 6 Jul 2020 18:35:50 -0600 Subject: [PATCH 10/37] Makefile: fix clean target recursive rm --- examples/fluids/shallow-water/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fluids/shallow-water/Makefile b/examples/fluids/shallow-water/Makefile index d9e2b9d419..c1a0b83b8a 100644 --- a/examples/fluids/shallow-water/Makefile +++ b/examples/fluids/shallow-water/Makefile @@ -78,7 +78,7 @@ print: $(PETSc.pc) $(ceed.pc) @true clean: - $(RM) r $(OBJDIR) shallowwater *.vtu + $(RM) -r $(OBJDIR) shallowwater *.vtu $(PETSc.pc): $(if $(wildcard $@),,$(error \ From b9a6181728712ae02b250f9df2da10d358d36a2e Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Wed, 8 Jul 2020 11:37:29 -0600 Subject: [PATCH 11/37] Debugging after changing ncompq --- examples/fluids/shallow-water/qfunctions/advection.h | 12 ++++++------ .../fluids/shallow-water/qfunctions/geostrophic.h | 6 +++--- examples/fluids/shallow-water/qfunctions/mass.h | 2 -- examples/fluids/shallow-water/shallowwater.c | 5 +++++ examples/fluids/shallow-water/src/setup.c | 6 ++++-- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/advection.h b/examples/fluids/shallow-water/qfunctions/advection.h index 59bdb26480..6f652cf798 100644 --- a/examples/fluids/shallow-water/qfunctions/advection.h +++ b/examples/fluids/shallow-water/qfunctions/advection.h @@ -82,11 +82,11 @@ CEED_QFUNCTION(ICsSW_Advection)(void *ctx, CeedInt Q, // Quadrature Point Loop for (CeedInt i=0; iqceed, NULL); CeedElemRestrictionCreateVector(Erestrictq, &user->qdotceed, NULL); CeedElemRestrictionCreateVector(Erestrictq, &user->gceed, NULL); + CeedElemRestrictionCreateVector(Erestrictq, &user->fceed, NULL); + CeedElemRestrictionCreateVector(Erestrictq, &user->jceed, NULL); // Create the explicit part of the PDE operator CeedOperatorCreate(ceed, qf_explicit, NULL, NULL, &op_explicit); @@ -420,7 +422,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_explicit, "dq", Erestrictq, basisq, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_explicit, "qdata", Erestrictq, + CeedOperatorSetField(op_explicit, "qdata", Erestrictqdi, CEED_BASIS_COLLOCATED, qdata); CeedOperatorSetField(op_explicit, "v", Erestrictq, basisq, CEED_VECTOR_ACTIVE); @@ -435,7 +437,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CeedOperatorSetField(op_implicit, "dq", Erestrictq, basisq, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_implicit, "qdot", Erestrictq, basisq, user->qdotceed); - CeedOperatorSetField(op_implicit, "qdata", Erestrictq, + CeedOperatorSetField(op_implicit, "qdata", Erestrictqdi, CEED_BASIS_COLLOCATED, qdata); CeedOperatorSetField(op_implicit, "x", Erestrictx, basisx, xcorners); CeedOperatorSetField(op_implicit, "v", Erestrictq, basisq, From 89caa26122dbc163e773bcc924233a7c2de2b503 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 13 Jul 2020 18:46:36 -0600 Subject: [PATCH 12/37] WIP: Jacobian for advection test case --- .../shallow-water/qfunctions/advection.h | 217 ++++++++++++------ .../shallow-water/qfunctions/geostrophic.h | 46 ++-- examples/fluids/shallow-water/shallowwater.c | 5 +- examples/fluids/shallow-water/src/setup.c | 51 ++-- examples/fluids/shallow-water/src/wrappers.c | 6 +- examples/fluids/shallow-water/sw_headers.h | 5 +- 6 files changed, 214 insertions(+), 116 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/advection.h b/examples/fluids/shallow-water/qfunctions/advection.h index 6f652cf798..7b791f096c 100644 --- a/examples/fluids/shallow-water/qfunctions/advection.h +++ b/examples/fluids/shallow-water/qfunctions/advection.h @@ -32,12 +32,12 @@ // ***************************************************************************** // This QFunction sets the the initial condition for the advection of a cosine -// bell shaped scalar function (test case 1 in "A Standard Test Set for -// Numerical Approximations to the Shallow Water Equations in Spherical +// bell shaped scalar function (test case 1 in "A Standard Test Set for +// Numerical Approximations to the Shallow Water Equations in Spherical // Geometry" by Williamson et al. (1992) // ***************************************************************************** -static inline int Exact_SW_Advection(CeedInt dim, CeedScalar time, - const CeedScalar X[], CeedInt Nf, +static inline int Exact_SW_Advection(CeedInt dim, CeedScalar time, + const CeedScalar X[], CeedInt Nf, CeedScalar q[], void *ctx) { // Context @@ -52,12 +52,12 @@ static inline int Exact_SW_Advection(CeedInt dim, CeedScalar time, // -- Compute latitude const CeedScalar theta = asin(X[2] / R); // -- Compute longitude - const CeedScalar lambda = atan2(X[1], X[0]); + const CeedScalar lambda = atan2(X[1], X[0]); // -- Compute great circle distance between (lambda, theta) and the center, // (lambda_c, theta_c) const CeedScalar lambda_c = 3. * M_PI / 2.; const CeedScalar theta_c = 0.; - const CeedScalar r = R * acos(sin(theta_c)*sin(theta) + cos(theta_c)*cos(theta)*cos(lambda-lambda_c)); + const CeedScalar r = R * acos(sin(theta_c)*sin(theta) + cos(theta_c)*cos(theta)*cos(lambda-lambda_c)); // Initial Conditions q[0] = u0 * (cos(theta)*cos(gamma) + sin(theta)*cos(lambda)*sin(gamma)); @@ -95,13 +95,13 @@ CEED_QFUNCTION(ICsSW_Advection)(void *ctx, CeedInt Q, } // ***************************************************************************** -// This QFunction implements the explicit terms of the advection test for the +// This QFunction implements the explicit terms of the advection test for the // shallow-water equations solver. // -// For this simple test, the equation represents the tranport of the scalar -// field h advected by the wind u, on a spherical surface, where the state -// variables, u_lambda, u_theta (or u_1, u_2) represent the longitudinal -// and latitudinal components of the velocity (wind) field, and h, represents +// For this simple test, the equation represents the tranport of the scalar +// field h advected by the wind u, on a spherical surface, where the state +// variables, u_lambda, u_theta (or u_1, u_2) represent the longitudinal +// and latitudinal components of the velocity (wind) field, and h, represents // the height function. // // State variable vector: q = (u_lambda, u_theta, h) @@ -114,7 +114,7 @@ CEED_QFUNCTION(ICsSW_Advection)(void *ctx, CeedInt Q, // // This QFunction has been adapted from navier-stokes/advection.h // ***************************************************************************** -CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, +CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* @@ -176,20 +176,20 @@ CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, // The Physics // No Change in velocity - for (CeedInt f=0; f<2; f++) { - for (CeedInt j=0; j<2; j++) + for (CeedInt f=0; f<2; f++) { // Equation/Component + for (CeedInt j=0; j<2; j++) // Derivative dv[j][f][i] = 0; v[f][i] = 0; } // -- Height: - // Evaluate the strong form using + // Evaluate the strong form using // div((h + H0) u) = u . grad(h) + (h + H0) div(u), with H0 constant // or in index notation: (u_j h)_{,j} = u_j h_j + (h + H0) u_{j,j} CeedScalar div_u = 0, u_dot_grad_h = 0; for (CeedInt j=0; j<2; j++) { CeedScalar dhdx_j = 0; - for (CeedInt k=0; k<2; k++) { + for (CeedInt k=0; k<2; k++) { // TODO: check indices! in this case dXdx is 2x3 div_u += du[j][k] * dXdx[k][j]; // u_{j,j} = u_{j,K} X_{K,j} dhdx_j += dh[k] * dXdx[k][j]; } @@ -199,7 +199,7 @@ CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, // Weak Galerkin convection term: dv \cdot ((h + H0) u) for (CeedInt j=0; j<2; j++) - dv[j][2][i] = (1 - strong_form) * wdetJ * (h + H0) * + dv[j][2][i] = (1 - strong_form) * wdetJ * (h + H0) * (u[0]*dXdx[j][0] + u[1]*dXdx[j][1]); v[2][i] = 0; @@ -209,7 +209,7 @@ CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, // Stabilization requires a measure of element transit time in the velocity // field u. CeedScalar uX[2]; - for (CeedInt j=0; j<2; j++) + for (CeedInt j=0; j<2; j++) uX[j] = dXdx[j][0]*u[0] + dXdx[j][1]*u[1]; const CeedScalar TauS = CtauS / sqrt(uX[0]*uX[0] + uX[1]*uX[1]); for (CeedInt j=0; j<2; j++) @@ -220,13 +220,13 @@ CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, } // ***************************************************************************** -// This QFunction implements the implicit terms of the advection test for the +// This QFunction implements the implicit terms of the advection test for the // shallow-water equations solver. // -// For this simple test, the equation represents the tranport of the scalar -// field h advected by the wind u, on a spherical surface, where the state -// variables, u_lambda, u_theta (or u_1, u_2) represent the longitudinal -// and latitudinal components of the velocity (wind) field, and h, represents +// For this simple test, the equation represents the tranport of the scalar +// field h advected by the wind u, on a spherical surface, where the state +// variables, u_lambda, u_theta (or u_1, u_2) represent the longitudinal +// and latitudinal components of the velocity (wind) field, and h, represents // the height function. // // State variable vector: q = (u_lambda, u_theta, h) @@ -240,7 +240,7 @@ CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, // To the spatial term F(t,q) one needs to add qdot (time derivative) on the LHS // This QFunction has been adapted from navier-stokes/advection.h // ***************************************************************************** -CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, +CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* @@ -297,10 +297,10 @@ CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, // dXdx[k][j] = dX_K / dx_j // X_K=Kth reference element coordinate (note cap X and K instead of xi_k} // x_j and u_j are jth physical position and velocity components - + // The Physics - - // No Change in velocity + + // No Change in velocity equations: for (CeedInt f=0; f<2; f++) { for (CeedInt j=0; j<2; j++) dv[j][f][i] = 0; @@ -314,7 +314,7 @@ CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, CeedScalar div_u = 0, u_dot_grad_h = 0; for (CeedInt j=0; j<2; j++) { CeedScalar dhdx_j = 0; - for (CeedInt k=0; k<2; k++) { + for (CeedInt k=0; k<2; k++) { // TODO: check indices! in this case dXdx is 2x3 div_u += du[j][k] * dXdx[k][j]; // u_{j,j} = u_{j,K} X_{K,j} dhdx_j += dh[k] * dXdx[k][j]; } @@ -322,24 +322,24 @@ CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, } CeedScalar strongConv = (h + H0) *div_u + u_dot_grad_h; CeedScalar strongResid = qdot[4][i] + strongConv; - + v[2][i] = wdetJ * qdot[2][i]; // transient part (ALWAYS) - + // Weak Galerkin convection term: -dv \cdot ((h + H0) u) for (CeedInt j=0; j<2; j++) - dv[j][2][i] = -wdetJ * (1 - strong_form) * (h + H0) * + dv[j][2][i] = -wdetJ * (1 - strong_form) * (h + H0) * (u[0]*dXdx[j][0] + u[1]*dXdx[j][1]); - + // Strong Galerkin convection term: v div((h + H0) u) v[2][i] += wdetJ * strong_form * strongConv; - + // Stabilization requires a measure of element transit time in the velocity // field u. CeedScalar uX[2]; - for (CeedInt j=0; j<2; j++) + for (CeedInt j=0; j<2; j++) uX[j] = dXdx[j][0]*u[0] + dXdx[j][1]*u[1]; const CeedScalar TauS = CtauS / sqrt(uX[0]*uX[0] + uX[1]*uX[1]); - + for (CeedInt j=0; j<2; j++) switch (context->stabilization) { case 0: @@ -350,80 +350,151 @@ CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, break; } } // End Quadrature Point Loop - + return 0; } // ***************************************************************************** -// This QFunction implements the Jacobian of of the advection test for the +// This QFunction implements the Jacobian of of the advection test for the // shallow-water equations solver. // -// For this simple test, the equation represents the tranport of the scalar -// field h advected by the wind u, on a spherical surface, where the state -// variables, u_lambda, u_theta (or u_1, u_2) represent the longitudinal -// and latitudinal components of the velocity (wind) field, and h, represents +// For this simple test, the equation represents the tranport of the scalar +// field h advected by the wind u, on a spherical surface, where the state +// variables, u_lambda, u_theta (or u_1, u_2) represent the longitudinal +// and latitudinal components of the velocity (wind) field, and h, represents // the height function. // -// Discrete Jacobian: +// Discrete Jacobian: // dF/dq^n = sigma * dF/dqdot|q^n + dF/dq|q^n // ("sigma * dF/dqdot|q^n" will be added later) // ***************************************************************************** -CEED_QFUNCTION(SWJacobian_Advection)(void *ctx, CeedInt Q, +CEED_QFUNCTION(SWJacobian_Advection)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs const CeedScalar (*q)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[0], - (*deltaq)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], - (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + (*dq)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*deltaq)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*deltadq)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[3], + (*qdata)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[4]; // Outputs - CeedScalar (*deltadvdX)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[0]; + CeedScalar (*deltav)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], + (*deltadvdX)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[1]; // *INDENT-ON* // Context ProblemContext context = (ProblemContext)ctx; - const CeedScalar H0 = context->H0; + const CeedScalar H0 = context->H0; + const CeedScalar CtauS = context->CtauS; + const CeedScalar strong_form = context->strong_form; CeedPragmaSIMD // Quadrature Point Loop for (CeedInt i=0; iqdatasize, + CeedInt numP, numQ, cStart, cEnd, nelem, qdatasize = problem->qdatasize, topodim = problem->topodim;; // CEED bases - P = degree + 1; - Q = P + qextra; - CeedBasisCreateTensorH1Lagrange(ceed, topodim, ncompq, P, Q, + numP = degree + 1; + numQ = numP + qextra; + CeedBasisCreateTensorH1Lagrange(ceed, topodim, ncompq, numP, numQ, CEED_GAUSS, &basisq); - CeedBasisCreateTensorH1Lagrange(ceed, topodim, ncompx, 2, Q, + CeedBasisCreateTensorH1Lagrange(ceed, topodim, ncompx, 2, numQ, CEED_GAUSS, &basisx); - CeedBasisCreateTensorH1Lagrange(ceed, topodim, ncompx, 2, P, + CeedBasisCreateTensorH1Lagrange(ceed, topodim, ncompx, 2, numP, CEED_GAUSS_LOBATTO, &basisxc); ierr = DMGetCoordinateDM(dm, &dmcoord); CHKERRQ(ierr); @@ -329,14 +329,15 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CHKERRQ(ierr); // CEED restrictions - ierr = CreateRestrictionPlex(ceed, dm, P, ncompq, &Erestrictq); CHKERRQ(ierr); + ierr = CreateRestrictionPlex(ceed, dm, numP, ncompq, &Erestrictq); + CHKERRQ(ierr); ierr = CreateRestrictionPlex(ceed, dmcoord, 2, ncompx, &Erestrictx); CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr); nelem = cEnd - cStart; - CeedElemRestrictionCreateStrided(ceed, nelem, Q*Q, qdatasize, - qdatasize*nelem*Q*Q, + CeedElemRestrictionCreateStrided(ceed, nelem, numQ*numQ, qdatasize, + qdatasize*nelem*numQ*numQ, CEED_STRIDES_BACKEND, &Erestrictqdi); // Element coordinates @@ -351,7 +352,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, user->q0ceed = q0ceed; // Create the Q-Function that builds the quadrature data - CeedQFunctionCreateInterior(ceed, 1, problem->setup, problem->setup_loc, + CeedQFunctionCreateInterior(ceed, 1, problem->setup, problem->setup_loc, &qf_setup); CeedQFunctionAddInput(qf_setup, "x", ncompx, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_setup, "dx", ncompx*topodim, CEED_EVAL_GRAD); @@ -364,7 +365,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CeedQFunctionAddOutput(qf_ics, "q0", ncompq, CEED_EVAL_NONE); // Create the Q-Function that defines the explicit part of the PDE operator - CeedQFunctionCreateInterior(ceed, 1, problem->apply_explfunction, + CeedQFunctionCreateInterior(ceed, 1, problem->apply_explfunction, problem->apply_explfunction_loc, &qf_explicit); CeedQFunctionAddInput(qf_explicit, "x", ncompx, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_explicit, "q", ncompq, CEED_EVAL_INTERP); @@ -374,7 +375,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CeedQFunctionAddOutput(qf_explicit, "dv", ncompq*topodim, CEED_EVAL_GRAD); // Create the Q-Function that defines the implicit part of the PDE operator - CeedQFunctionCreateInterior(ceed, 1, problem->apply_implfunction, + CeedQFunctionCreateInterior(ceed, 1, problem->apply_implfunction, problem->apply_implfunction_loc, &qf_implicit); CeedQFunctionAddInput(qf_implicit, "q", ncompq, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_implicit, "dq", ncompq*topodim, CEED_EVAL_GRAD); @@ -385,12 +386,16 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CeedQFunctionAddOutput(qf_implicit, "dv", ncompq*topodim, CEED_EVAL_GRAD); // Create the Q-Function that defines the action of the Jacobian operator - CeedQFunctionCreateInterior(ceed, 1, problem->apply_jacobian, + CeedQFunctionCreateInterior(ceed, 1, problem->apply_jacobian, problem->apply_jacobian_loc, &qf_jacobian); - CeedQFunctionAddInput(qf_jacobian, "q", 3, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_jacobian, "deltaq", 3, CEED_EVAL_NONE); - CeedQFunctionAddInput(qf_jacobian, "qdata", 10, CEED_EVAL_NONE); - CeedQFunctionAddOutput(qf_jacobian, "dv", 3, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_jacobian, "q", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_jacobian, "dq", ncompq*topodim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_jacobian, "deltaq", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_jacobian, "deltadq", ncompq*topodim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_jacobian, "qdata", qdatasize, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_jacobian, "deltav", ncompq, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_jacobian, "deltadv", ncompq*topodim, + CEED_EVAL_GRAD); // Create the operator that builds the quadrature data for the operator CeedOperatorCreate(ceed, qf_setup, NULL, NULL, &op_setup); @@ -450,11 +455,17 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CeedOperatorCreate(ceed, qf_jacobian, NULL, NULL, &op_jacobian); CeedOperatorSetField(op_jacobian, "q", Erestrictq, basisq, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dq", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_jacobian, "deltaq", Erestrictq, basisq, - CEED_VECTOR_ACTIVE); // TODO: Check restriction and basis for delta q + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "deltadq", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_jacobian, "qdata", Erestrictqdi, CEED_BASIS_COLLOCATED, qdata); - CeedOperatorSetField(op_jacobian, "dv", Erestrictq, basisq, + CeedOperatorSetField(op_jacobian, "deltav", Erestrictq, basisq, + CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "deltadv", Erestrictq, basisq, CEED_VECTOR_ACTIVE); user->op_jacobian = op_jacobian; diff --git a/examples/fluids/shallow-water/src/wrappers.c b/examples/fluids/shallow-water/src/wrappers.c index 5d4c0aa873..8104bd10c4 100644 --- a/examples/fluids/shallow-water/src/wrappers.c +++ b/examples/fluids/shallow-water/src/wrappers.c @@ -129,10 +129,10 @@ PetscErrorCode FormJacobian_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, PetscErrorCode ierr; PetscFunctionBeginUser; - - ierr = TSComputeIJacobianDefaultColor(ts, t, Q, Qdot, sigma, J, Jpre, NULL); + + ierr = TSComputeIJacobianDefaultColor(ts, t, Q, Qdot, sigma, J, Jpre, NULL); CHKERRQ(ierr); - + // Jpre might be AIJ (e.g., when using coloring), so we need to assemble it ierr = MatAssemblyBegin(Jpre, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = MatAssemblyEnd(Jpre, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index 6b216b5c88..5019c80bc4 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -42,6 +42,7 @@ typedef struct { typedef PhysicsContext_s *PhysicsContext; typedef struct { + CeedScalar g; CeedScalar H0; CeedScalar CtauS; CeedScalar strong_form; @@ -152,8 +153,8 @@ PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, CeedInt ncomp, CeedElemRestriction *Erestrict); // Auxiliary function to set up libCEED objects for a given degree -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, - PetscInt ncompx, PetscInt ncompq, User user, +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, + PetscInt ncompx, PetscInt ncompq, User user, CeedData data, problemData *problem, PhysicsContext phys_ctx, ProblemContext probl_ctx); From b6545dff23fc3b30806c77ad0caf0662747a126f Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Thu, 16 Jul 2020 17:03:48 -0600 Subject: [PATCH 13/37] SWE: Assign DM label 'panel' for different faces of initial cube --- examples/fluids/shallow-water/shallowwater.c | 22 ++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 5a1487e15e..683f044fef 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -80,7 +80,7 @@ int main(int argc, char **argv) { PetscScalar R_e = 6.37122e6; // Earth radius (m) PetscScalar g = 9.81; // gravitational acceleration (m/s^2) PetscScalar H0 = 0; // constant mean height (m) - PetscScalar gamma = 0; // angle between axis of rotation and polar axis + PetscScalar gamma = 0; // angle between axis of rotation and polar axis PetscScalar mpersquareds; // Check PETSc CUDA support PetscBool petschavecuda, setmemtyperequest = PETSC_FALSE; @@ -209,7 +209,7 @@ int main(int argc, char **argv) { .gamma = gamma, .time = 0. }; - + ProblemContext_s probl_ctx = { .g = g, .H0 = H0, @@ -229,6 +229,16 @@ int main(int argc, char **argv) { CHKERRQ(ierr); // Set the object name ierr = PetscObjectSetName((PetscObject)dm, "Sphere"); CHKERRQ(ierr); + // Define cube panels (charts) + DMLabel label; + PetscInt c, cStart, cEnd, npanel; + ierr = DMCreateLabel(dm, "panel"); + ierr = DMGetLabel(dm, "panel", &label); + // Assign different panel (chart) values to the six faces of the cube + ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr); + for (c = cStart, npanel = 0; c < cEnd; c++) { + ierr = DMLabelSetValue(label, c, npanel++); CHKERRQ(ierr); + } // Distribute mesh over processes { DM dmDist = NULL; @@ -366,7 +376,7 @@ int main(int argc, char **argv) { // Setup libCEED's objects ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); - ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, ceeddata, + ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, ceeddata, problem, &phys_ctx, &probl_ctx); CHKERRQ(ierr); // Set up PETSc context @@ -403,7 +413,7 @@ int main(int argc, char **argv) { ierr = ICs_FixMultiplicity(ceeddata->op_ics, ceeddata->xcorners, user->q0ceed, dm, Qloc, Q, ceeddata->Erestrictq, &phys_ctx, 0.0); CHKERRQ(ierr); - + MPI_Comm_rank(comm, &rank); if (!rank) { ierr = PetscMkdir(user->outputfolder); CHKERRQ(ierr); @@ -425,8 +435,8 @@ int main(int argc, char **argv) { ierr = DMRestoreLocalVector(dm, &Qloc); CHKERRQ(ierr); // Set up the MatShell for the associated Jacobian operator - ierr = MatCreateShell(PETSC_COMM_SELF, ncompq*odofs, ncompq*odofs, - PETSC_DETERMINE, PETSC_DETERMINE, user, &J); + ierr = MatCreateShell(PETSC_COMM_SELF, ncompq*odofs, ncompq*odofs, + PETSC_DETERMINE, PETSC_DETERMINE, user, &J); CHKERRQ(ierr); // Set the MatShell operation needed for the Jacobian ierr = MatShellSetOperation(J, MATOP_MULT, From ca8aa06f82b50e085c0aa77ce6cd03dec19f5e5c Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Thu, 16 Jul 2020 20:18:59 -0600 Subject: [PATCH 14/37] SWE: Add FindPanelEdgeNodes() function to determine which nodes are on the cube edges The algorithm with bit manipulation was greatly inspired-by and helped-by @matteovigni Thanks-to @matteovigni --- .../shallow-water/qfunctions/setup_geo.h | 2 +- examples/fluids/shallow-water/shallowwater.c | 6 +- examples/fluids/shallow-water/src/setup.c | 83 +++++++++++++++++-- examples/fluids/shallow-water/sw_headers.h | 14 ++++ 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/setup_geo.h b/examples/fluids/shallow-water/qfunctions/setup_geo.h index 3949c10e50..b79a85083b 100644 --- a/examples/fluids/shallow-water/qfunctions/setup_geo.h +++ b/examples/fluids/shallow-water/qfunctions/setup_geo.h @@ -171,7 +171,7 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, qdata[7][i] = dxdXTdxdXinv[0][0]; qdata[8][i] = dxdXTdxdXinv[1][1]; qdata[9][i] = dxdXTdxdXinv[0][1]; - + // Terrain topography, hs qdata[10][i] = sin(xx[0]) + cos(xx[1]); // put 0 for constant flat topography } // End of Quadrature Point Loop diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 683f044fef..4e769f9fa2 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -56,6 +56,7 @@ int main(int argc, char **argv) { Units units; problemType problemChoice; problemData *problem = NULL; + EdgeNode edgenodes; StabilizationType stab; PetscBool implicit; PetscInt degree, qextra, outputfreq, steps, contsteps; @@ -65,7 +66,7 @@ int main(int argc, char **argv) { const CeedInt ncompx = 3; PetscInt viz_refine = 0; PetscBool read_mesh, simplex, test; - PetscInt topodim = 2, ncompq = 3, lnodes; + PetscInt topodim = 2, ncompq = 3, lnodes, nedgenodes; // libCEED context char ceedresource[PETSC_MAX_PATH_LEN] = "/cpu/self", filename[PETSC_MAX_PATH_LEN]; @@ -374,6 +375,9 @@ int main(int argc, char **argv) { // Set up global mass vector ierr = VecDuplicate(Q, &user->M); CHKERRQ(ierr); + // Setup global lat-long vector for different panels (charts) of the cube + ierr = FindPanelEdgeNodes(dm, ncompq, &nedgenodes, &edgenodes); CHKERRQ(ierr); + // Setup libCEED's objects ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, ceeddata, diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 61dbe6b2c2..8fa6f82db7 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,78 @@ problemData problemOptions[] = { } }; +// ----------------------------------------------------------------------------- +// Auxiliary function to determine if nodes belong to cube faces (panels) +// ----------------------------------------------------------------------------- + +PetscErrorCode FindPanelEdgeNodes(DM dm, PetscInt ncomp, PetscInt *nedgenodes, + EdgeNode *edgenodes) { + + PetscInt ierr; + PetscInt cstart, cend, nstart, nend, nnodes, depth, edgenode = 0; + PetscSection section; + PetscFunctionBeginUser; + // Get Nelem + ierr = DMGetSection(dm, §ion); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(dm, 0, &cstart, &cend); CHKERRQ(ierr); + ierr = DMPlexGetDepth(dm, &depth); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(dm, depth, &nstart, &nend); CHKERRQ(ierr); + nnodes = nend - nstart; + unsigned int bitmap[nnodes]; + ierr = PetscMemzero(bitmap, sizeof(bitmap)); CHKERRQ(ierr); + + // Get indices + for (PetscInt c = cstart; c < cend; c++) { // Traverse elements + PetscInt numindices, *indices, n, panel; + // Query element panel + ierr = DMGetLabelValue(dm, "panel", c, &panel); CHKERRQ(ierr); + ierr = DMPlexGetClosureIndices(dm, section, section, c, PETSC_TRUE, + &numindices, &indices, NULL, NULL); + CHKERRQ(ierr); + for (n = 0; n < numindices; n += ncomp) { // Traverse nodes per element + PetscInt bitmapidx = indices[n] / ncomp; + bitmap[bitmapidx] |= (1 << panel); + } + ierr = DMPlexRestoreClosureIndices(dm, section, section, c, PETSC_TRUE, + &numindices, &indices, NULL, NULL); + CHKERRQ(ierr); + } + // Read the 1's in the resulting bitmap and extract edge nodes only + ierr = PetscMalloc1(nnodes + 24, edgenodes); CHKERRQ(ierr); + for (PetscInt i = 0; i < nnodes; i++) { + PetscInt ones = 0, panels[3]; + for (PetscInt p = 0; p < 6; p++) { + if (bitmap[i] & 1) + panels[ones++] = p; + bitmap[i] >>= 1; + } + if (ones == 2) { + (*edgenodes)[edgenode].idx = i; + (*edgenodes)[edgenode].panelA = panels[0]; + (*edgenodes)[edgenode].panelB = panels[1]; + edgenode++; + } + else if (ones == 3) { + (*edgenodes)[edgenode].idx = i; + (*edgenodes)[edgenode].panelA = panels[0]; + (*edgenodes)[edgenode].panelB = panels[1]; + edgenode++; + (*edgenodes)[edgenode].idx = i; + (*edgenodes)[edgenode].panelA = panels[0]; + (*edgenodes)[edgenode].panelB = panels[2]; + edgenode++; + (*edgenodes)[edgenode].idx = i; + (*edgenodes)[edgenode].panelA = panels[1]; + (*edgenodes)[edgenode].panelB = panels[2]; + edgenode++; + } + } + ierr = PetscRealloc(edgenode, edgenodes); CHKERRQ(ierr); + *nedgenodes = edgenode; + + PetscFunctionReturn(0); +} + // ----------------------------------------------------------------------------- // Auxiliary function to create PETSc FE space for a given degree // ----------------------------------------------------------------------------- @@ -178,7 +251,7 @@ PetscErrorCode SetupDM(DM dm, PetscInt degree, PetscInt ncompq, // PETSc sphere auxiliary function // ----------------------------------------------------------------------------- -// Utility function taken from petsc/src/dm/impls/plex/examples/tutorials/ex7.c +// Utility function taken from petsc/src/dm/impls/plex/tutorials/ex7.c PetscErrorCode ProjectToUnitSphere(DM dm) { Vec coordinates; PetscScalar *coords; @@ -223,13 +296,13 @@ PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, // Get indices ierr = PetscMalloc1(nelem*P*P, &erestrict); CHKERRQ(ierr); - for (c=cStart, eoffset = 0; c Date: Thu, 16 Jul 2020 20:20:49 -0600 Subject: [PATCH 15/37] Update comment in area and bpssphere setups --- examples/petsc/setuparea.h | 2 +- examples/petsc/setupsphere.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/petsc/setuparea.h b/examples/petsc/setuparea.h index df2a7ecae0..5897b74c89 100644 --- a/examples/petsc/setuparea.h +++ b/examples/petsc/setuparea.h @@ -121,7 +121,7 @@ static problemData problemOptions[6] = { // PETSc sphere auxiliary functions // ----------------------------------------------------------------------------- -// Utility function taken from petsc/src/dm/impls/plex/examples/tutorials/ex7.c +// Utility function taken from petsc/src/dm/impls/plex/tutorials/ex7.c static PetscErrorCode ProjectToUnitSphere(DM dm) { Vec coordinates; PetscScalar *coords; diff --git a/examples/petsc/setupsphere.h b/examples/petsc/setupsphere.h index 6820308f54..f585979bf0 100644 --- a/examples/petsc/setupsphere.h +++ b/examples/petsc/setupsphere.h @@ -219,7 +219,7 @@ static bpData bpOptions[6] = { // PETSc sphere auxiliary functions // ----------------------------------------------------------------------------- -// Utility function taken from petsc/src/dm/impls/plex/examples/tutorials/ex7.c +// Utility function taken from petsc/src/dm/impls/plex/tutorials/ex7.c static PetscErrorCode ProjectToUnitSphere(DM dm) { Vec coordinates; PetscScalar *coords; From d6a97a4b7e39e3472273f7785fd583bde1e50927 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 20 Jul 2020 23:25:44 -0600 Subject: [PATCH 16/37] SWE: Define coordinate transformations for points expressed with local coordinate systems on different panels (faces) of the cube --- .../shallow-water/qfunctions/setup_geo.h | 23 +- examples/fluids/shallow-water/shallowwater.c | 7 +- examples/fluids/shallow-water/src/setup.c | 264 +++++++++++++++--- examples/fluids/shallow-water/sw_headers.h | 14 +- 4 files changed, 251 insertions(+), 57 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/setup_geo.h b/examples/fluids/shallow-water/qfunctions/setup_geo.h index b79a85083b..144791987e 100644 --- a/examples/fluids/shallow-water/qfunctions/setup_geo.h +++ b/examples/fluids/shallow-water/qfunctions/setup_geo.h @@ -108,13 +108,13 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, }; CeedScalar dxdX[3][2]; - for (int j=0; j<3; j++) - for (int k=0; k<2; k++) { + for (CeedInt j=0; j<3; j++) { + for (CeedInt k=0; k<2; k++) { dxdX[j][k] = 0; - for (int l=0; l<3; l++) + for (CeedInt l=0; l<3; l++) dxdX[j][k] += dxdxx[j][l]*dxxdX[l][k]; } - + } // J is given by the cross product of the columns of dxdX const CeedScalar J[3] = {dxdX[1][0]*dxdX[2][1] - dxdX[2][0]*dxdX[1][1], dxdX[2][0]*dxdX[0][1] - dxdX[0][0]*dxdX[2][1], @@ -129,13 +129,13 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, // dxdX_k,j * dxdX_j,k CeedScalar dxdXTdxdX[2][2]; - for (int j=0; j<2; j++) - for (int k=0; k<2; k++) { + for (CeedInt j=0; j<2; j++) { + for (CeedInt k=0; k<2; k++) { dxdXTdxdX[j][k] = 0; - for (int l=0; l<3; l++) + for (CeedInt l=0; l<3; l++) dxdXTdxdX[j][k] += dxdX[l][j]*dxdX[l][k]; } - + } const CeedScalar detdxdXTdxdX = dxdXTdxdX[0][0] * dxdXTdxdX[1][1] -dxdXTdxdX[1][0] * dxdXTdxdX[0][1]; @@ -151,12 +151,13 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, // Compute the pseudo inverse of dxdX CeedScalar pseudodXdx[2][3]; - for (int j=0; j<2; j++) - for (int k=0; k<3; k++) { + for (CeedInt j=0; j<2; j++) { + for (CeedInt k=0; k<3; k++) { pseudodXdx[j][k] = 0; - for (int l=0; l<2; l++) + for (CeedInt l=0; l<2; l++) pseudodXdx[j][k] += dxdXTdxdXinv[j][l]*dxdX[k][l]; } + } // Interp-to-Grad qdata // Pseudo inverse of dxdX: (x_i,j)+ = X_i,j diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 4e769f9fa2..4f6feaff66 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -56,7 +56,6 @@ int main(int argc, char **argv) { Units units; problemType problemChoice; problemData *problem = NULL; - EdgeNode edgenodes; StabilizationType stab; PetscBool implicit; PetscInt degree, qextra, outputfreq, steps, contsteps; @@ -66,7 +65,7 @@ int main(int argc, char **argv) { const CeedInt ncompx = 3; PetscInt viz_refine = 0; PetscBool read_mesh, simplex, test; - PetscInt topodim = 2, ncompq = 3, lnodes, nedgenodes; + PetscInt topodim = 2, ncompq = 3, lnodes; // libCEED context char ceedresource[PETSC_MAX_PATH_LEN] = "/cpu/self", filename[PETSC_MAX_PATH_LEN]; @@ -369,14 +368,14 @@ int main(int argc, char **argv) { cEnd - cStart, gdofs/ncompq, odofs/ncompq, ncompq, gdofs, odofs); CHKERRQ(ierr); } - } // Set up global mass vector ierr = VecDuplicate(Q, &user->M); CHKERRQ(ierr); // Setup global lat-long vector for different panels (charts) of the cube - ierr = FindPanelEdgeNodes(dm, ncompq, &nedgenodes, &edgenodes); CHKERRQ(ierr); + Mat T; + ierr = FindPanelEdgeNodes(dm, &phys_ctx, ncompq, &T); CHKERRQ(ierr); // Setup libCEED's objects ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 8fa6f82db7..8af283d78f 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -71,22 +71,39 @@ problemData problemOptions[] = { // Auxiliary function to determine if nodes belong to cube faces (panels) // ----------------------------------------------------------------------------- -PetscErrorCode FindPanelEdgeNodes(DM dm, PetscInt ncomp, PetscInt *nedgenodes, - EdgeNode *edgenodes) { +PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, + PetscInt ncomp, Mat *T) { - PetscInt ierr; - PetscInt cstart, cend, nstart, nend, nnodes, depth, edgenode = 0; + PetscInt ierr, rankid; + MPI_Comm comm; + PetscInt cstart, cend, nstart, nend, lnodes, gdofs, depth, edgenodecnt = 0; PetscSection section; + EdgeNode edgenodes; + PetscFunctionBeginUser; // Get Nelem - ierr = DMGetSection(dm, §ion); CHKERRQ(ierr); + ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cstart, &cend); CHKERRQ(ierr); ierr = DMPlexGetDepth(dm, &depth); CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, depth, &nstart, &nend); CHKERRQ(ierr); - nnodes = nend - nstart; - unsigned int bitmap[nnodes]; + lnodes = nend - nstart; + + PetscSF sf; + Vec bitmapVec; + ierr = DMGetSectionSF(dm, &sf); CHKERRQ(ierr); + ierr = DMCreateGlobalVector(dm, &bitmapVec); CHKERRQ(ierr); + ierr = VecGetSize(bitmapVec, &gdofs); CHKERRQ(ierr); + ierr = VecDestroy(&bitmapVec); + + // Arrays for local and global bitmaps + unsigned int bitmaploc[lnodes * ncomp]; + unsigned int bitmap[gdofs]; ierr = PetscMemzero(bitmap, sizeof(bitmap)); CHKERRQ(ierr); + // Broadcast bitmap values to all ranks + ierr = PetscSFBcastBegin(sf, MPI_UNSIGNED, bitmap, bitmaploc); CHKERRQ(ierr); + ierr = PetscSFBcastEnd(sf, MPI_UNSIGNED, bitmap, bitmaploc); CHKERRQ(ierr); + // Get indices for (PetscInt c = cstart; c < cend; c++) { // Traverse elements PetscInt numindices, *indices, n, panel; @@ -96,49 +113,220 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PetscInt ncomp, PetscInt *nedgenodes, &numindices, &indices, NULL, NULL); CHKERRQ(ierr); for (n = 0; n < numindices; n += ncomp) { // Traverse nodes per element - PetscInt bitmapidx = indices[n] / ncomp; - bitmap[bitmapidx] |= (1 << panel); + PetscInt bitmapidx = indices[n]; + bitmaploc[bitmapidx] |= (1 << panel); } ierr = DMPlexRestoreClosureIndices(dm, section, section, c, PETSC_TRUE, &numindices, &indices, NULL, NULL); CHKERRQ(ierr); } - // Read the 1's in the resulting bitmap and extract edge nodes only - ierr = PetscMalloc1(nnodes + 24, edgenodes); CHKERRQ(ierr); - for (PetscInt i = 0; i < nnodes; i++) { - PetscInt ones = 0, panels[3]; - for (PetscInt p = 0; p < 6; p++) { - if (bitmap[i] & 1) - panels[ones++] = p; - bitmap[i] >>= 1; - } - if (ones == 2) { - (*edgenodes)[edgenode].idx = i; - (*edgenodes)[edgenode].panelA = panels[0]; - (*edgenodes)[edgenode].panelB = panels[1]; - edgenode++; + + // Reduce on all ranks + ierr = PetscSFReduceBegin(sf, MPI_UNSIGNED, bitmaploc, bitmap, MPI_BOR); + CHKERRQ(ierr); + ierr = PetscSFReduceEnd(sf, MPI_UNSIGNED, bitmaploc, bitmap, MPI_BOR); + CHKERRQ(ierr); + + // Rank 0 reads the 1's in the resulting bitmap and extracts edge nodes only + PetscObjectGetComm((PetscObject)dm, &comm); CHKERRQ(ierr); + MPI_Comm_rank(comm, &rankid); + if (rankid == 0) { + ierr = PetscMalloc1(gdofs + 24*ncomp, &edgenodes); CHKERRQ(ierr); + for (PetscInt i = 0; i < gdofs; i += ncomp) { + PetscInt ones = 0, panels[3]; + for (PetscInt p = 0; p < 6; p++) { + if (bitmap[i] & 1) + panels[ones++] = p; + bitmap[i] >>= 1; + } + if (ones == 2) { + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[0]; + edgenodes[edgenodecnt].panelB = panels[1]; + edgenodecnt++; + } + else if (ones == 3) { + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[0]; + edgenodes[edgenodecnt].panelB = panels[1]; + edgenodecnt++; + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[0]; + edgenodes[edgenodecnt].panelB = panels[2]; + edgenodecnt++; + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[1]; + edgenodes[edgenodecnt].panelB = panels[2]; + edgenodecnt++; + } } - else if (ones == 3) { - (*edgenodes)[edgenode].idx = i; - (*edgenodes)[edgenode].panelA = panels[0]; - (*edgenodes)[edgenode].panelB = panels[1]; - edgenode++; - (*edgenodes)[edgenode].idx = i; - (*edgenodes)[edgenode].panelA = panels[0]; - (*edgenodes)[edgenode].panelB = panels[2]; - edgenode++; - (*edgenodes)[edgenode].idx = i; - (*edgenodes)[edgenode].panelA = panels[1]; - (*edgenodes)[edgenode].panelB = panels[2]; - edgenode++; + ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, edgenodes, + edgenodecnt, T); CHKERRQ(ierr); + // Free edgenodes structure array + ierr = PetscFree(edgenodes); CHKERRQ(ierr); + } + + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// Auxiliary function that sets up all corrdinate transformations between panels +// ----------------------------------------------------------------------------- +PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, + PetscInt ncomp, + EdgeNode edgenodes, + PetscInt nedgenodes, Mat *T) { + PetscInt ierr; + MPI_Comm comm; + Vec X; + PetscInt gdofs; + const PetscScalar *xarray; + + PetscFunctionBeginUser; + ierr = PetscObjectGetComm((PetscObject)dm, &comm); CHKERRQ(ierr); + + ierr = DMGetCoordinates(dm, &X); CHKERRQ(ierr); + ierr = VecGetSize(X, &gdofs); CHKERRQ(ierr); + + // Preallocate sparse matrix + ierr = MatCreateBAIJ(comm, 2, PETSC_DECIDE, PETSC_DECIDE, 4*gdofs/ncomp, + 4*gdofs/ncomp, 2, NULL, 0, NULL, T); CHKERRQ(ierr); + for (PetscInt i=0; i < 4*gdofs/ncomp; i++) { + ierr = MatSetValue(*T, i, i, 1., INSERT_VALUES); CHKERRQ(ierr); + } + + ierr = VecGetArrayRead(X, &xarray); CHKERRQ(ierr); + PetscScalar R = phys_ctx->R; + // Nodes loop + for (PetscInt i=0; i < gdofs; i += ncomp) { + // Read global Cartesian coordinates + PetscScalar x[3] = {xarray[i*ncomp + 0], + xarray[i*ncomp + 1], + xarray[i*ncomp + 2] + }; + // Normalize quadrature point coordinates to sphere + PetscScalar rad = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); + x[0] *= R / rad; + x[1] *= R / rad; + x[2] *= R / rad; + // Compute latitude and longitude + const PetscScalar theta = asin(x[2] / R); // latitude + const PetscScalar lambda = atan2(x[1], x[0]); // longitude + + // For P_1 (east), P_3 (front), P_4 (west), P_5 (back): + PetscScalar T00 = cos(theta)*cos(lambda) * cos(lambda); + PetscScalar T01 = cos(theta)*cos(lambda) * 0.; + PetscScalar T10 = cos(theta)*cos(lambda) * -sin(theta)*sin(lambda); + PetscScalar T11 = cos(theta)*cos(lambda) * cos(theta); + const PetscScalar T_lateral[2][2] = {{T00, + T01}, + {T10, + T11} + }; + PetscScalar Tinv00 = 1./(cos(theta)*cos(lambda)) * (1./cos(lambda)); + PetscScalar Tinv01 = 1./(cos(theta)*cos(lambda)) * 0.; + PetscScalar Tinv10 = 1./(cos(theta)*cos(lambda)) * tan(theta)*tan(lambda); + PetscScalar Tinv11 = 1./(cos(theta)*cos(lambda)) * (1./cos(theta)); + const PetscScalar T_lateralinv[2][2] = {{Tinv00, + Tinv01}, + {Tinv10, + Tinv11} + }; + // For P2 (north): + T00 = sin(theta) * cos(lambda); + T01 = sin(theta) * sin(lambda); + T10 = sin(theta) * -sin(theta)*sin(lambda); + T11 = sin(theta) * sin(theta)*cos(lambda); + const PetscScalar T_top[2][2] = {{T00, + T01}, + {T10, + T11} + }; + Tinv00 = 1./(sin(theta)*sin(theta)) * sin(theta)*cos(lambda); + Tinv01 = 1./(sin(theta)*sin(theta)) * (-sin(lambda)); + Tinv10 = 1./(sin(theta)*sin(theta)) * sin(theta)*sin(lambda); + Tinv11 = 1./(sin(theta)*sin(theta)) * cos(lambda); + const PetscScalar T_topinv[2][2] = {{Tinv00, + Tinv01}, + {Tinv10, + Tinv11} + }; + + // For P0 (south): + T00 = sin(theta) * (-cos(theta)); + T01 = sin(theta) * sin(lambda); + T10 = sin(theta) * sin(theta)*sin(lambda); + T11 = sin(theta) * sin(theta)*cos(lambda); + const PetscScalar T_bottom[2][2] = {{T00, + T01}, + {T10, + T11} + }; + Tinv00 = 1./(sin(theta)*sin(theta)) * (-sin(theta)*cos(lambda)); + Tinv01 = 1./(sin(theta)*sin(theta)) * sin(lambda); + Tinv10 = 1./(sin(theta)*sin(theta)) * sin(theta)*sin(lambda); + Tinv11 = 1./(sin(theta)*sin(theta)) * cos(lambda); + const PetscScalar T_bottominv[2][2] = {{Tinv00, + Tinv01}, + {Tinv10, + Tinv11} + }; + + const PetscScalar (*transforms[6])[2][2] = {&T_bottom, + &T_lateral, + &T_top, + &T_lateral, + &T_lateral, + &T_lateral + }; + + const PetscScalar (*inv_transforms[6])[2][2] = {&T_bottominv, + &T_lateralinv, + &T_topinv, + &T_lateralinv, + &T_lateralinv, + &T_lateralinv + }; + + for (PetscInt e = 0; e < nedgenodes; e++) { + if (edgenodes[e].idx == i) { + const PetscScalar (*matrixA)[2][2] = inv_transforms[edgenodes[e].panelA]; + const PetscScalar (*matrixB)[2][2] = transforms[edgenodes[e].panelB]; + + // inv_transform * transform (A^{-1}*B) + // This product represents the mapping from coordinate system A + // to spherical coordinates and then to coordinate system B. Vice versa + // for transform * inv_transform (B^{-1}*A) + PetscScalar matrixAB[2][2], matrixBA[2][2]; + for (int j=0; j<2; j++) { + for (int k=0; k<2; k++) { + matrixAB[j][k] = matrixBA[j][k] = 0; + for (int l=0; l<2; l++) { + matrixAB[j][k] += (*matrixA)[j][l] * (*matrixB)[l][k]; + matrixBA[j][k] += (*matrixB)[j][l] * (*matrixA)[l][k]; + } + } + } + PetscInt idxAB[2] = {4*i/ncomp + 0, 4*i/ncomp +1}; + PetscInt idxBA[2] = {4*i/ncomp + 2, 4*i/ncomp +3}; + ierr = MatSetValues(*T, 2, idxAB, 2, idxAB, (PetscScalar *)matrixAB, + INSERT_VALUES); CHKERRQ(ierr); + ierr = MatSetValues(*T, 2, idxBA, 2, idxBA, (PetscScalar *)matrixBA, + INSERT_VALUES); CHKERRQ(ierr); + } } } - ierr = PetscRealloc(edgenode, edgenodes); CHKERRQ(ierr); - *nedgenodes = edgenode; + // Assemble matrix for all node transformations + ierr = MatAssemblyBegin(*T, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); + ierr = MatAssemblyEnd(*T, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); + + // Restore array read + ierr = VecRestoreArrayRead(X, &xarray); CHKERRQ(ierr); PetscFunctionReturn(0); } + // ----------------------------------------------------------------------------- // Auxiliary function to create PETSc FE space for a given degree // ----------------------------------------------------------------------------- diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index a101a9d112..f0ad6489d5 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -119,7 +119,7 @@ struct Units_ { }; struct EdgeNode_ { - PetscInt idx; // Node index + PetscInt idx; // Node index PetscInt panelA, panelB; // Indices of panels sharing the edge node }; @@ -142,9 +142,15 @@ extern problemData problemOptions[]; // Auxiliary functions for cube face (panel) charts // ----------------------------------------------------------------------------- -// Auxiliary function to determine if nodes belong to cube faces (panels) -PetscErrorCode FindPanelEdgeNodes(DM dm, PetscInt ncomp, PetscInt *nedgenodes, - EdgeNode *edgenodes); +// Auxiliary function to determine if nodes belong to cube face (panel) edges +PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, + PetscInt ncomp, Mat *T); + +// Auxiliary function that sets up all corrdinate transformations between panels +PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, + PetscInt ncomp, + EdgeNode edgenodes, + PetscInt nedgenodes, Mat *T); // ----------------------------------------------------------------------------- // Setup DM functions From b7ae1a09e925de98e1870f39aff2dc9f7183e690 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 21 Jul 2020 10:55:25 -0600 Subject: [PATCH 17/37] Add TESTARGS and update sample runs --- examples/fluids/shallow-water/qfunctions/geostrophic.h | 2 +- examples/fluids/shallow-water/shallowwater.c | 8 +++----- examples/fluids/shallow-water/src/setup.c | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/geostrophic.h b/examples/fluids/shallow-water/qfunctions/geostrophic.h index 72a5fa5ea9..6229c920fb 100644 --- a/examples/fluids/shallow-water/qfunctions/geostrophic.h +++ b/examples/fluids/shallow-water/qfunctions/geostrophic.h @@ -133,7 +133,7 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // *INDENT-OFF* // Setup // -- Compute latitude - const CeedScalar theta = asin(X[2][i] / R); + const CeedScalar theta = asin(X[2][i] / R); // -- Compute Coriolis parameter const CeedScalar f = 2*Omega*sin(theta); // -- Interp in diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 4f6feaff66..393f1ae621 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -30,12 +30,10 @@ // Sample runs: // // shallow-water -// shallow-water -ceed /cpu/self -// shallow-water -ceed /gpu/occa -// shallow-water -ceed /cpu/occa -// shallow-water -ceed /omp/occa -// shallow-water -ceed /ocl/occa +// shallow-water -ceed /cpu/self -dm_refine 1 snes_fd_color -ts_fd_color -degree 1 +// shallow-water -ceed /gpu/occa -dm_refine 1 snes_fd_color -ts_fd_color -degree 1 // +//TESTARGS -ceed {ceed_resource} -test -dm_refine 1 snes_fd_color -ts_fd_color -degree 1 /// @file /// Shallow-water equations example using PETSc diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 8af283d78f..6739d712a1 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -190,7 +190,7 @@ PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, // Preallocate sparse matrix ierr = MatCreateBAIJ(comm, 2, PETSC_DECIDE, PETSC_DECIDE, 4*gdofs/ncomp, - 4*gdofs/ncomp, 2, NULL, 0, NULL, T); CHKERRQ(ierr); + 4*gdofs/ncomp, 2, NULL, 0, NULL, T); CHKERRQ(ierr); for (PetscInt i=0; i < 4*gdofs/ncomp; i++) { ierr = MatSetValue(*T, i, i, 1., INSERT_VALUES); CHKERRQ(ierr); } From 7b8a94a63f458ff56f53460245ef6532d5e76dae Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 27 Jul 2020 18:53:31 -0600 Subject: [PATCH 18/37] SWE: Change panel ordering, following papers convention --- examples/fluids/shallow-water/shallowwater.c | 4 ++-- examples/fluids/shallow-water/src/setup.c | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 393f1ae621..dae61168a3 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -229,13 +229,13 @@ int main(int argc, char **argv) { ierr = PetscObjectSetName((PetscObject)dm, "Sphere"); CHKERRQ(ierr); // Define cube panels (charts) DMLabel label; - PetscInt c, cStart, cEnd, npanel; + PetscInt c, cStart, cEnd, npanel, permidx[6] = {5, 1, 4, 0, 3, 2}; ierr = DMCreateLabel(dm, "panel"); ierr = DMGetLabel(dm, "panel", &label); // Assign different panel (chart) values to the six faces of the cube ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr); for (c = cStart, npanel = 0; c < cEnd; c++) { - ierr = DMLabelSetValue(label, c, npanel++); CHKERRQ(ierr); + ierr = DMLabelSetValue(label, c, permidx[npanel++]); CHKERRQ(ierr); } // Distribute mesh over processes { diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 6739d712a1..12e6b83df6 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -213,7 +213,7 @@ PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, const PetscScalar theta = asin(x[2] / R); // latitude const PetscScalar lambda = atan2(x[1], x[0]); // longitude - // For P_1 (east), P_3 (front), P_4 (west), P_5 (back): + // For P_0 (front), P_1 (east), P_2 (back), P_3 (west): PetscScalar T00 = cos(theta)*cos(lambda) * cos(lambda); PetscScalar T01 = cos(theta)*cos(lambda) * 0.; PetscScalar T10 = cos(theta)*cos(lambda) * -sin(theta)*sin(lambda); @@ -232,7 +232,7 @@ PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, {Tinv10, Tinv11} }; - // For P2 (north): + // For P4 (north): T00 = sin(theta) * cos(lambda); T01 = sin(theta) * sin(lambda); T10 = sin(theta) * -sin(theta)*sin(lambda); @@ -252,7 +252,7 @@ PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, Tinv11} }; - // For P0 (south): + // For P5 (south): T00 = sin(theta) * (-cos(theta)); T01 = sin(theta) * sin(lambda); T10 = sin(theta) * sin(theta)*sin(lambda); @@ -272,20 +272,20 @@ PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, Tinv11} }; - const PetscScalar (*transforms[6])[2][2] = {&T_bottom, + const PetscScalar (*transforms[6])[2][2] = {&T_lateral, &T_lateral, - &T_top, &T_lateral, &T_lateral, - &T_lateral + &T_top, + &T_bottom }; - const PetscScalar (*inv_transforms[6])[2][2] = {&T_bottominv, + const PetscScalar (*inv_transforms[6])[2][2] = {&T_lateralinv, &T_lateralinv, - &T_topinv, &T_lateralinv, &T_lateralinv, - &T_lateralinv + &T_topinv, + &T_bottominv }; for (PetscInt e = 0; e < nedgenodes; e++) { From e0f578e86e9b655c27f506ad663fa66796e2cb63 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 27 Jul 2020 23:33:56 -0600 Subject: [PATCH 19/37] SWE: Fix bit manipulation algorithm in parallel --- examples/fluids/shallow-water/shallowwater.c | 2 +- examples/fluids/shallow-water/src/setup.c | 108 +++++++++---------- 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index dae61168a3..d878e43663 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -436,7 +436,7 @@ int main(int argc, char **argv) { ierr = DMRestoreLocalVector(dm, &Qloc); CHKERRQ(ierr); // Set up the MatShell for the associated Jacobian operator - ierr = MatCreateShell(PETSC_COMM_SELF, ncompq*odofs, ncompq*odofs, + ierr = MatCreateShell(comm, ncompq*odofs, ncompq*odofs, PETSC_DETERMINE, PETSC_DETERMINE, user, &J); CHKERRQ(ierr); // Set the MatShell operation needed for the Jacobian diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 12e6b83df6..becf302b3b 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -74,19 +74,18 @@ problemData problemOptions[] = { PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, PetscInt ncomp, Mat *T) { - PetscInt ierr, rankid; + PetscInt ierr; MPI_Comm comm; - PetscInt cstart, cend, nstart, nend, lnodes, gdofs, depth, edgenodecnt = 0; - PetscSection section; + PetscInt cstart, cend, gdofs, depth, edgenodecnt = 0; + PetscSection section, sectionloc; EdgeNode edgenodes; PetscFunctionBeginUser; // Get Nelem - ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); + ierr = DMGetGlobalSection(dm, §ion); CHKERRQ(ierr); + ierr = DMGetLocalSection(dm, §ionloc); CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cstart, &cend); CHKERRQ(ierr); ierr = DMPlexGetDepth(dm, &depth); CHKERRQ(ierr); - ierr = DMPlexGetHeightStratum(dm, depth, &nstart, &nend); CHKERRQ(ierr); - lnodes = nend - nstart; PetscSF sf; Vec bitmapVec; @@ -96,75 +95,70 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, ierr = VecDestroy(&bitmapVec); // Arrays for local and global bitmaps - unsigned int bitmaploc[lnodes * ncomp]; - unsigned int bitmap[gdofs]; - ierr = PetscMemzero(bitmap, sizeof(bitmap)); CHKERRQ(ierr); - - // Broadcast bitmap values to all ranks - ierr = PetscSFBcastBegin(sf, MPI_UNSIGNED, bitmap, bitmaploc); CHKERRQ(ierr); - ierr = PetscSFBcastEnd(sf, MPI_UNSIGNED, bitmap, bitmaploc); CHKERRQ(ierr); + unsigned int *bitmaploc; + unsigned int *bitmap; + ierr = PetscCalloc2(gdofs, &bitmaploc, gdofs, &bitmap); + CHKERRQ(ierr); // Get indices for (PetscInt c = cstart; c < cend; c++) { // Traverse elements PetscInt numindices, *indices, n, panel; // Query element panel ierr = DMGetLabelValue(dm, "panel", c, &panel); CHKERRQ(ierr); - ierr = DMPlexGetClosureIndices(dm, section, section, c, PETSC_TRUE, + ierr = DMPlexGetClosureIndices(dm, sectionloc, section, c, PETSC_TRUE, &numindices, &indices, NULL, NULL); CHKERRQ(ierr); for (n = 0; n < numindices; n += ncomp) { // Traverse nodes per element - PetscInt bitmapidx = indices[n]; + PetscInt bitmapidx = indices[n]; bitmaploc[bitmapidx] |= (1 << panel); } - ierr = DMPlexRestoreClosureIndices(dm, section, section, c, PETSC_TRUE, + ierr = DMPlexRestoreClosureIndices(dm, sectionloc, section, c, PETSC_TRUE, &numindices, &indices, NULL, NULL); CHKERRQ(ierr); } - // Reduce on all ranks - ierr = PetscSFReduceBegin(sf, MPI_UNSIGNED, bitmaploc, bitmap, MPI_BOR); - CHKERRQ(ierr); - ierr = PetscSFReduceEnd(sf, MPI_UNSIGNED, bitmaploc, bitmap, MPI_BOR); - CHKERRQ(ierr); - - // Rank 0 reads the 1's in the resulting bitmap and extracts edge nodes only + // Reduce from all ranks PetscObjectGetComm((PetscObject)dm, &comm); CHKERRQ(ierr); - MPI_Comm_rank(comm, &rankid); - if (rankid == 0) { - ierr = PetscMalloc1(gdofs + 24*ncomp, &edgenodes); CHKERRQ(ierr); - for (PetscInt i = 0; i < gdofs; i += ncomp) { - PetscInt ones = 0, panels[3]; - for (PetscInt p = 0; p < 6; p++) { - if (bitmap[i] & 1) - panels[ones++] = p; - bitmap[i] >>= 1; - } - if (ones == 2) { - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[0]; - edgenodes[edgenodecnt].panelB = panels[1]; - edgenodecnt++; - } - else if (ones == 3) { - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[0]; - edgenodes[edgenodecnt].panelB = panels[1]; - edgenodecnt++; - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[0]; - edgenodes[edgenodecnt].panelB = panels[2]; - edgenodecnt++; - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[1]; - edgenodes[edgenodecnt].panelB = panels[2]; - edgenodecnt++; - } + MPI_Reduce(bitmaploc, bitmap, gdofs, MPI_UNSIGNED, MPI_BOR, 0, comm); + + // Read the resulting bitmap and extract edge nodes only + ierr = PetscMalloc1(gdofs + 24*ncomp, &edgenodes); CHKERRQ(ierr); + for (PetscInt i = 0; i < gdofs; i += ncomp) { + PetscInt ones = 0, panels[3]; + for (PetscInt p = 0; p < 6; p++) { + if (bitmap[i] & 1) + panels[ones++] = p; + bitmap[i] >>= 1; + } + if (ones == 2) { + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[0]; + edgenodes[edgenodecnt].panelB = panels[1]; + edgenodecnt++; + } + else if (ones == 3) { + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[0]; + edgenodes[edgenodecnt].panelB = panels[1]; + edgenodecnt++; + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[0]; + edgenodes[edgenodecnt].panelB = panels[2]; + edgenodecnt++; + edgenodes[edgenodecnt].idx = i; + edgenodes[edgenodecnt].panelA = panels[1]; + edgenodes[edgenodecnt].panelB = panels[2]; + edgenodecnt++; } - ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, edgenodes, - edgenodecnt, T); CHKERRQ(ierr); - // Free edgenodes structure array - ierr = PetscFree(edgenodes); CHKERRQ(ierr); } + ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, edgenodes, + edgenodecnt, T); CHKERRQ(ierr); + + // Free edgenodes structure array + ierr = PetscFree(edgenodes); CHKERRQ(ierr); + // Free heap + ierr = PetscFree(bitmap); CHKERRQ(ierr); + ierr = PetscFree(bitmaploc); CHKERRQ(ierr); PetscFunctionReturn(0); } From 29de1ea6d7b116d9a2a754f3c7c7792a2f08dc2a Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 4 Aug 2020 14:45:52 -0600 Subject: [PATCH 20/37] SWE: Update DMPlexCreateSphereMesh() call after change in PETSc's API and add compatibility macro --- .../shallow-water/qfunctions/geostrophic.h | 20 +++++++++---------- examples/fluids/shallow-water/shallowwater.c | 3 ++- examples/fluids/shallow-water/src/setup.c | 4 ++++ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/geostrophic.h b/examples/fluids/shallow-water/qfunctions/geostrophic.h index 6229c920fb..a2fa2bd33d 100644 --- a/examples/fluids/shallow-water/qfunctions/geostrophic.h +++ b/examples/fluids/shallow-water/qfunctions/geostrophic.h @@ -83,9 +83,9 @@ CEED_QFUNCTION(ICsSW)(void *ctx, CeedInt Q, const CeedScalar x[] = {X[0][i], X[1][i], X[2][i]}; CeedScalar q[5]; - Exact_SW(2, 0., x, 5, q, ctx); + Exact_SW(2, 0., x, 3, q, ctx); - for (CeedInt j=0; j<5; j++) + for (CeedInt j=0; j<3; j++) q0[j][i] = q[j]; } // End of Quadrature Point Loop @@ -153,7 +153,7 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // The Physics // Explicit spatial terms of G_1(t,q): // Explicit terms multiplying v - // - (omega + f) * khat curl u - grad(|u|^2/2) + // - (omega + f) * khat curl u - grad(|u|^2/2) // TODO: needs fix with weak form v[0][i] = - wdetJ*(u[0]*du[0][0] + u[1]*du[0][1] + f*u[1]); // No explicit terms multiplying dv dv[0][0][i] = 0; @@ -161,7 +161,7 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Explicit spatial terms of G_2(t,q): // Explicit terms multiplying v - // - (omega + f) * khat curl u - grad(|u|^2/2) + // - (omega + f) * khat curl u - grad(|u|^2/2) // TODO: needs fix with weak form v[1][i] = - wdetJ*(u[0]*du[1][0] + u[1]*du[1][1] - f*u[0]); // No explicit terms multiplying dv dv[0][1][i] = 0; @@ -210,9 +210,9 @@ CEED_QFUNCTION(SWImplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, (*dv)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[1]; // *INDENT-ON* // Context - const PhysicsContext context = (PhysicsContext)ctx; - const CeedScalar g = context->g; - const CeedScalar H0 = context->H0; + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar g = context->g; + const CeedScalar H0 = context->H0; CeedPragmaSIMD // Quadrature Point Loop @@ -299,9 +299,9 @@ CEED_QFUNCTION(SWJacobian)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar (*deltadvdX)[3][CEED_Q_VLA] = (CeedScalar(*)[3][CEED_Q_VLA])out[0]; // *INDENT-ON* // Context - const PhysicsContext context = (PhysicsContext)ctx; - const CeedScalar g = context->g; - const CeedScalar H0 = context->H0; + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar g = context->g; + const CeedScalar H0 = context->H0; CeedPragmaSIMD // Quadrature Point Loop diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index d878e43663..1e2a68e9a6 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -223,7 +223,8 @@ int main(int argc, char **argv) { } else { // Create the mesh as a 0-refined sphere. This will create a cubic surface, not a box. PetscBool simplex = PETSC_FALSE; - ierr = DMPlexCreateSphereMesh(PETSC_COMM_WORLD, topodim, simplex, &dm); + ierr = DMPlexCreateSphereMesh(PETSC_COMM_WORLD, topodim, simplex, + phys_ctx.R, &dm); CHKERRQ(ierr); // Set the object name ierr = PetscObjectSetName((PetscObject)dm, "Sphere"); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index becf302b3b..9dc30ae836 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -34,6 +34,10 @@ # define DMPlexRestoreClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexRestoreClosureIndices(a,b,c,d,f,g,i) #endif +#if PETSC_VERSION_LT(3,14,0) +# define DMPlexCreateSphereMesh(a,b,c,d,e) DMPlexCreateSphereMesh(a,b,c,e) +#endif + problemData problemOptions[] = { [SWE_ADVECTION] = { .topodim = 2, From f205e8519536c525d84d292f13ac193e9901524a Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Fri, 14 Aug 2020 16:39:25 -0600 Subject: [PATCH 21/37] Bpssphere: improve comment --- examples/petsc/bpssphere.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/petsc/bpssphere.c b/examples/petsc/bpssphere.c index 7536c6e5c0..cafea96b79 100644 --- a/examples/petsc/bpssphere.c +++ b/examples/petsc/bpssphere.c @@ -117,7 +117,7 @@ int main(int argc, char **argv) { NULL, simplex, &simplex, NULL); CHKERRQ(ierr); ierr = PetscOptionsEnd(); CHKERRQ(ierr); - // Setup DM + // Create DM if (read_mesh) { ierr = DMPlexCreateFromFile(PETSC_COMM_WORLD, filename, PETSC_TRUE, &dm); CHKERRQ(ierr); @@ -148,7 +148,7 @@ int main(int argc, char **argv) { ierr = DMViewFromOptions(dm, NULL, "-dm_view"); CHKERRQ(ierr); } - // Create DM + // Setup DM ierr = SetupDMByDegree(dm, degree, ncompu, topodim); CHKERRQ(ierr); // Create vectors From 578b94b1298d9b54684d2488378355d0d1930db7 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Fri, 14 Aug 2020 16:39:54 -0600 Subject: [PATCH 22/37] SWE: WIP Improve comment and change in geom factors QFunction --- .../shallow-water/qfunctions/setup_geo.h | 75 ++++++++++++------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/setup_geo.h b/examples/fluids/shallow-water/qfunctions/setup_geo.h index 144791987e..2befcb6e44 100644 --- a/examples/fluids/shallow-water/qfunctions/setup_geo.h +++ b/examples/fluids/shallow-water/qfunctions/setup_geo.h @@ -26,35 +26,44 @@ // ***************************************************************************** // This QFunction sets up the geometric factors required for integration and -// coordinate transformations +// coordinate transformations. See ref: "A Discontinuous Galerkin Transport +// Scheme on the Cubed Sphere", by Nair et al. (2004). // -// Reference (parent) 2D coordinates: X \in [-1, 1]^2 +// Reference (parent) 2D coordinates: X \in [-1, 1]^2. // -// Global 3D physical coordinates given by the mesh: xx \in [-R, R]^3 -// with R radius of the sphere +// Local 2D physical coordinates on the 2D manifold: x \in [-l, l]^2 +// with l half edge of the cube inscribed in the sphere. These coordinate +// systems vary locally on each face (or panel) of the cube. // -// Local 3D physical coordinates on the 2D manifold: x \in [-l, l]^3 -// with l half edge of the cube inscribed in the sphere +// (theta, lambda) represnt latitude and longitude coordinates. // -// Change of coordinates matrix computed by the library: -// (physical 3D coords relative to reference 2D coords) -// dxx_j/dX_i (indicial notation) [3 * 2] +// Change of coordinates from x (on the 2D manifold) to xx (phyisical 3D on +// the sphere), i.e., "cube-to-sphere" A, with equidistant central projection: // -// Change of coordinates x (on the 2D manifold) relative to xx (phyisical 3D): -// dx_i/dxx_j (indicial notation) [3 * 3] +// For lateral panels (P0-P3): +// A = R cos(theta)cos(lambda) / l * [cos(lambda) 0] +// [-sin(theta)sin(lambda) cos(theta)] // -// Change of coordinates x (on the 2D manifold) relative to X (reference 2D): -// (by chain rule) -// dx_i/dX_j [3 * 2] = dx_i/dxx_k [3 * 3] * dxx_k/dX_j [3 * 2] +// For top panel (P4): +// A = R sin(theta) / l * [cos(lambda) sin(lambda)] +// [-sin(theta)sin(lambda) sin(theta)cos(lambda)] // -// modJ is given by the magnitude of the cross product of the columns of dx_i/dX_j +// For bottom panel(P5): +// A = R sin(theta) / l * [-cos(lambda) sin(lambda)] +// [sin(theta)sin(lambda) sin(theta)cos(lambda)] +// +// The inverse of A, A^{-1}, is the "sphere-to-cube" change of coordinates. +// +// The metric tensor g_{ij} = A^TA, and its inverse, +// g^{ij} = g_{ij}^{-1} = A^{-1}A^{-T} +// +// modJ represents the magnitude of the cross product of the columns of A, i.e., +// J = det(g_{ij}) // // The quadrature data is stored in the array qdata. // // We require the determinant of the Jacobian to properly compute integrals of -// the form: int( u v ) -// -// Qdata: modJ * w +// the form: int( u v ). // // ***************************************************************************** CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, @@ -69,13 +78,25 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, // *INDENT-ON* CeedPragmaSIMD - // Quadrature Point Loop + // Quadrature Point Loop to determine on which panel the element belongs to for (CeedInt i=0; i Date: Fri, 14 Aug 2020 16:41:23 -0600 Subject: [PATCH 23/37] SWE: Transform 3D global coords to 2D local panel coords and update QFunctionSetContenxt after PR#596 --- examples/fluids/shallow-water/shallowwater.c | 63 ++++--- examples/fluids/shallow-water/src/setup.c | 172 +++++++++++++++---- examples/fluids/shallow-water/sw_headers.h | 22 ++- 3 files changed, 193 insertions(+), 64 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 1e2a68e9a6..4a11f828fb 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -56,11 +56,11 @@ int main(int argc, char **argv) { problemData *problem = NULL; StabilizationType stab; PetscBool implicit; - PetscInt degree, qextra, outputfreq, steps, contsteps; + PetscInt degree, qextra, outputfreq, steps, contsteps, nedgenodes = 0; PetscMPIInt rank; PetscScalar ftime; - Vec Q, Qloc, Xloc; - const CeedInt ncompx = 3; + Vec Q, Qloc, Xloc, Xpanelsloc; + const PetscInt ncompx = 3; PetscInt viz_refine = 0; PetscBool read_mesh, simplex, test; PetscInt topodim = 2, ncompq = 3, lnodes; @@ -196,7 +196,7 @@ int main(int argc, char **argv) { g *= mpersquareds; // Set up the libCEED context - PhysicsContext_s phys_ctx = { + PhysicsContext_s physCtxData = { .u0 = 0., .v0 = 0., .h0 = .1, @@ -208,7 +208,7 @@ int main(int argc, char **argv) { .time = 0. }; - ProblemContext_s probl_ctx = { + ProblemContext_s problCtxData = { .g = g, .H0 = H0, .CtauS = CtauS, @@ -216,7 +216,7 @@ int main(int argc, char **argv) { .stabilization = stab }; - // Setup DM + // Create DM if (read_mesh) { ierr = DMPlexCreateFromFile(PETSC_COMM_WORLD, filename, PETSC_TRUE, &dm); CHKERRQ(ierr); @@ -224,11 +224,11 @@ int main(int argc, char **argv) { // Create the mesh as a 0-refined sphere. This will create a cubic surface, not a box. PetscBool simplex = PETSC_FALSE; ierr = DMPlexCreateSphereMesh(PETSC_COMM_WORLD, topodim, simplex, - phys_ctx.R, &dm); + physCtxData.R, &dm); CHKERRQ(ierr); // Set the object name ierr = PetscObjectSetName((PetscObject)dm, "Sphere"); CHKERRQ(ierr); - // Define cube panels (charts) + // Define cube panels DMLabel label; PetscInt c, cStart, cEnd, npanel, permidx[6] = {5, 1, 4, 0, 3, 2}; ierr = DMCreateLabel(dm, "panel"); @@ -238,6 +238,7 @@ int main(int argc, char **argv) { for (c = cStart, npanel = 0; c < cEnd; c++) { ierr = DMLabelSetValue(label, c, permidx[npanel++]); CHKERRQ(ierr); } + ierr = DMPlexLabelComplete(dm, label); CHKERRQ(ierr); // Distribute mesh over processes { DM dmDist = NULL; @@ -259,11 +260,8 @@ int main(int argc, char **argv) { ierr = DMViewFromOptions(dm, NULL, "-dm_view"); CHKERRQ(ierr); } - // Create DM - ierr = DMLocalizeCoordinates(dm); CHKERRQ(ierr); - ierr = DMSetFromOptions(dm); CHKERRQ(ierr); - ierr = SetupDM(dm, degree, ncompq, topodim); - CHKERRQ(ierr); + // Setup DM + ierr = SetupDMByDegree(dm, degree, ncompq, topodim); CHKERRQ(ierr); // Refine DM for high-order viz dmviz = NULL; @@ -283,7 +281,8 @@ int main(int argc, char **argv) { ierr = DMSetCoarseDM(dmhierarchy[i+1], dmhierarchy[i]); CHKERRQ(ierr); d = (d + 1) / 2; if (i + 1 == viz_refine) d = 1; - ierr = SetupDM(dmhierarchy[i+1], degree, ncompq, topodim); CHKERRQ(ierr); + ierr = SetupDMByDegree(dmhierarchy[i+1], degree, ncompq, topodim); + CHKERRQ(ierr); ierr = DMCreateInterpolation(dmhierarchy[i], dmhierarchy[i+1], &interp_next, NULL); CHKERRQ(ierr); if (!i) interpviz = interp_next; @@ -296,7 +295,7 @@ int main(int argc, char **argv) { interpviz = C; } } - for (PetscInt i=1; iM); CHKERRQ(ierr); - // Setup global lat-long vector for different panels (charts) of the cube + // Setup transformation matrix different panels of the cube Mat T; - ierr = FindPanelEdgeNodes(dm, &phys_ctx, ncompq, &T); CHKERRQ(ierr); + EdgeNode edgenodes; + ierr = FindPanelEdgeNodes(dm, &physCtxData, ncompq, &nedgenodes, &edgenodes, + &T); + CHKERRQ(ierr); + + // Pass transformation matrix to context for setup operator + problCtxData.T = T; + + ierr = DMGetCoordinatesLocal(dm, &Xloc); CHKERRQ(ierr); + ierr = VecDuplicate(Xloc, &Xpanelsloc); CHKERRQ(ierr); + // Transform coordinate according to their panel systems + ierr = TransformCoords(dm, Xloc, ncompx, edgenodes, nedgenodes, &physCtxData, + &Xpanelsloc); CHKERRQ(ierr); + + // Free edgenodes structure array + ierr = PetscFree(edgenodes); CHKERRQ(ierr); // Setup libCEED's objects ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); - ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, ceeddata, - problem, &phys_ctx, &probl_ctx); CHKERRQ(ierr); + ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, + Xpanelsloc, ceeddata, problem, &physCtxData, &problCtxData); + CHKERRQ(ierr); // Set up PETSc context // Set up units structure @@ -402,9 +417,11 @@ int main(int argc, char **argv) { ierr = VecZeroEntries(Q); CHKERRQ(ierr); ierr = VectorPlacePetscVec(user->q0ceed, Qloc); CHKERRQ(ierr); + // Set up contex for QFunctions + CeedQFunctionSetContext(ceeddata->qf_setup, ceeddata->physCtx); + // Apply Setup Ceed Operators - ierr = DMGetCoordinatesLocal(dm, &Xloc); CHKERRQ(ierr); - ierr = VectorPlacePetscVec(ceeddata->xcorners, Xloc); CHKERRQ(ierr); + ierr = VectorPlacePetscVec(ceeddata->xcorners, Xpanelsloc); CHKERRQ(ierr); CeedOperatorApply(ceeddata->op_setup, ceeddata->xcorners, ceeddata->qdata, CEED_REQUEST_IMMEDIATE); ierr = ComputeLumpedMassMatrix(ceed, dm, ceeddata->Erestrictq, @@ -414,7 +431,7 @@ int main(int argc, char **argv) { // Apply IC operator and fix multiplicity of initial state vector ierr = ICs_FixMultiplicity(ceeddata->op_ics, ceeddata->xcorners, user->q0ceed, dm, Qloc, Q, ceeddata->Erestrictq, - &phys_ctx, 0.0); CHKERRQ(ierr); + &physCtxData, 0.0); CHKERRQ(ierr); MPI_Comm_rank(comm, &rank); if (!rank) { @@ -524,6 +541,8 @@ int main(int argc, char **argv) { CeedQFunctionDestroy(&ceeddata->qf_explicit); CeedQFunctionDestroy(&ceeddata->qf_implicit); CeedQFunctionDestroy(&ceeddata->qf_jacobian); + CeedQFunctionContextDestroy(&ceeddata->physCtx); + CeedQFunctionContextDestroy(&ceeddata->problCtx); CeedOperatorDestroy(&ceeddata->op_setup); CeedOperatorDestroy(&ceeddata->op_ics); CeedOperatorDestroy(&ceeddata->op_explicit); diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 9dc30ae836..d59087931e 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -27,7 +27,7 @@ #include "../sw_headers.h" // Function prototytes #include "../qfunctions/setup_geo.h" // Geometric factors #include "../qfunctions/advection.h" // Physics point-wise functions -#include "../qfunctions/geostrophic.h" // Physics point-wise functions +#include "../qfunctions/geostrophic.h" // Physics point-wise functions #if PETSC_VERSION_LT(3,14,0) # define DMPlexGetClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexGetClosureIndices(a,b,c,d,f,g,i) @@ -76,20 +76,19 @@ problemData problemOptions[] = { // ----------------------------------------------------------------------------- PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, - PetscInt ncomp, Mat *T) { + PetscInt ncomp, PetscInt *edgenodecnt, + EdgeNode *edgenodes, Mat *T) { PetscInt ierr; MPI_Comm comm; - PetscInt cstart, cend, gdofs, depth, edgenodecnt = 0; + PetscInt cstart, cend, gdofs; PetscSection section, sectionloc; - EdgeNode edgenodes; PetscFunctionBeginUser; // Get Nelem ierr = DMGetGlobalSection(dm, §ion); CHKERRQ(ierr); ierr = DMGetLocalSection(dm, §ionloc); CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cstart, &cend); CHKERRQ(ierr); - ierr = DMPlexGetDepth(dm, &depth); CHKERRQ(ierr); PetscSF sf; Vec bitmapVec; @@ -126,7 +125,7 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, MPI_Reduce(bitmaploc, bitmap, gdofs, MPI_UNSIGNED, MPI_BOR, 0, comm); // Read the resulting bitmap and extract edge nodes only - ierr = PetscMalloc1(gdofs + 24*ncomp, &edgenodes); CHKERRQ(ierr); + ierr = PetscMalloc1(gdofs + 24*ncomp, edgenodes); CHKERRQ(ierr); for (PetscInt i = 0; i < gdofs; i += ncomp) { PetscInt ones = 0, panels[3]; for (PetscInt p = 0; p < 6; p++) { @@ -135,31 +134,29 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, bitmap[i] >>= 1; } if (ones == 2) { - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[0]; - edgenodes[edgenodecnt].panelB = panels[1]; - edgenodecnt++; + (*edgenodes)[*edgenodecnt].idx = i; + (*edgenodes)[*edgenodecnt].panelA = panels[0]; + (*edgenodes)[*edgenodecnt].panelB = panels[1]; + (*edgenodecnt)++; } else if (ones == 3) { - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[0]; - edgenodes[edgenodecnt].panelB = panels[1]; - edgenodecnt++; - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[0]; - edgenodes[edgenodecnt].panelB = panels[2]; - edgenodecnt++; - edgenodes[edgenodecnt].idx = i; - edgenodes[edgenodecnt].panelA = panels[1]; - edgenodes[edgenodecnt].panelB = panels[2]; - edgenodecnt++; + (*edgenodes)[*edgenodecnt].idx = i; + (*edgenodes)[*edgenodecnt].panelA = panels[0]; + (*edgenodes)[*edgenodecnt].panelB = panels[1]; + (*edgenodecnt)++; + (*edgenodes)[*edgenodecnt].idx = i; + (*edgenodes)[*edgenodecnt].panelA = panels[0]; + (*edgenodes)[*edgenodecnt].panelB = panels[2]; + (*edgenodecnt)++; + (*edgenodes)[*edgenodecnt].idx = i; + (*edgenodes)[*edgenodecnt].panelA = panels[1]; + (*edgenodes)[*edgenodecnt].panelB = panels[2]; + (*edgenodecnt)++; } } - ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, edgenodes, - edgenodecnt, T); CHKERRQ(ierr); + ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, *edgenodes, + *edgenodecnt, T); CHKERRQ(ierr); - // Free edgenodes structure array - ierr = PetscFree(edgenodes); CHKERRQ(ierr); // Free heap ierr = PetscFree(bitmap); CHKERRQ(ierr); ierr = PetscFree(bitmaploc); CHKERRQ(ierr); @@ -168,7 +165,7 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, } // ----------------------------------------------------------------------------- -// Auxiliary function that sets up all corrdinate transformations between panels +// Auxiliary function that sets up all coordinate transformations between panels // ----------------------------------------------------------------------------- PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, PetscInt ncomp, @@ -324,6 +321,103 @@ PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, PetscFunctionReturn(0); } +// ----------------------------------------------------------------------------- +// Auxiliary function that converts global 3D coors into local panel coords +// ----------------------------------------------------------------------------- + +PetscErrorCode TransformCoords(DM dm, Vec Xloc, const PetscInt ncompx, + EdgeNode edgenodes, const PetscInt nedgenodes, + PhysicsContext phys_ctx, Vec *Xpanelsloc) { + + PetscInt ierr; + PetscInt lsize, depth, nstart, nend; + const PetscScalar *xlocarray; + PetscScalar *xpanelslocarray; + + PetscFunctionBeginUser; + ierr = VecGetArrayRead(Xloc, &xlocarray); CHKERRQ(ierr); + ierr = VecGetSize(Xloc, &lsize); CHKERRQ(ierr); + ierr = VecGetArray(*Xpanelsloc, &xpanelslocarray); CHKERRQ(ierr); + + ierr = DMPlexGetDepth(dm, &depth); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(dm, depth, &nstart, &nend); CHKERRQ(ierr); + + for (PetscInt n = 0; n < (nend - nstart); n++) { // Traverse nodes + // Query element panel + PetscInt panel; + PetscSection sectionloc; + ierr = DMGetLocalSection(dm, §ionloc); CHKERRQ(ierr); + ierr = DMGetLabelValue(dm, "panel", n + nstart, &panel); CHKERRQ(ierr); + + PetscScalar X = xlocarray[n*ncompx+0]; + PetscScalar Y = xlocarray[n*ncompx+1]; + PetscScalar Z = xlocarray[n*ncompx+2]; + + // Normalize quadrature point coordinates to sphere + CeedScalar rad = sqrt(X*X + Y*Y + Z*Z); + X *= phys_ctx->R / rad; + Y *= phys_ctx->R / rad; + Z *= phys_ctx->R / rad; + + PetscScalar x, y; + PetscScalar l = phys_ctx->R/sqrt(3.); + PetscBool isedgenode = PETSC_FALSE; + + // Determine if this node is an edge node + for (PetscInt e = 0; e < nedgenodes; e++) { + if (n * ncompx == edgenodes[e].idx) { + isedgenode = PETSC_TRUE; break; + } + } + + if (isedgenode) { + // Compute latitude and longitude + const CeedScalar theta = asin(Z / phys_ctx->R); // latitude + const CeedScalar lambda = atan2(Y, X); // longitude + xpanelslocarray[n*ncompx+0] = theta; + xpanelslocarray[n*ncompx+1] = lambda; + xpanelslocarray[n*ncompx+2] = -1; + } + + else { + + switch (panel) { + case 0: + x = l * (Y / X); + y = l * (Z / X); + break; + case 1: + x = l * (-X / Y); + y = l * (Z / Y); + break; + case 2: + x = l * (Y / X); + y = l * (-Z / X); + break; + case 3: + x = l * (-X / Y); + y = l * (-Z / Y); + break; + case 4: + x = l * (Y / Z); + y = l * (-X / Z); + break; + case 5: + x = l * (-Y / Z); + y = l * (-X / Z); + break; + } + xpanelslocarray[n*ncompx+0] = x; + xpanelslocarray[n*ncompx+1] = y; + xpanelslocarray[n*ncompx+2] = panel; + } + } // End of nodes for loop + + ierr = VecRestoreArrayRead(Xloc, &xlocarray); CHKERRQ(ierr); + ierr = VecRestoreArray(*Xpanelsloc, &xpanelslocarray); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} // ----------------------------------------------------------------------------- // Auxiliary function to create PETSc FE space for a given degree @@ -400,8 +494,8 @@ PetscErrorCode PetscFECreateByDegree(DM dm, PetscInt dim, PetscInt Nc, // Auxiliary function to setup DM FE space and info // ----------------------------------------------------------------------------- -PetscErrorCode SetupDM(DM dm, PetscInt degree, PetscInt ncompq, - PetscInt dim) { +PetscErrorCode SetupDMByDegree(DM dm, PetscInt degree, PetscInt ncompq, + PetscInt dim) { PetscErrorCode ierr; PetscFunctionBeginUser; @@ -558,11 +652,11 @@ PetscErrorCode VectorPlacePetscVec(CeedVector c, Vec p) { PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, PetscInt ncompx, PetscInt ncompq, User user, - CeedData data, problemData *problem, - PhysicsContext phys_ctx, ProblemContext probl_ctx) { + Vec Xloc, CeedData data, problemData *problem, + PhysicsContext physCtxData, + ProblemContext problCtxData) { int ierr; DM dmcoord; - Vec Xloc; CeedBasis basisx, basisxc, basisq; CeedElemRestriction Erestrictx, Erestrictq, Erestrictqdi; CeedQFunction qf_setup, qf_ics, qf_explicit, qf_implicit, @@ -729,10 +823,16 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, user->op_jacobian = op_jacobian; // Set up the libCEED context - CeedQFunctionSetContext(qf_ics, phys_ctx, sizeof *phys_ctx); - CeedQFunctionSetContext(qf_explicit, probl_ctx, sizeof *probl_ctx); - CeedQFunctionSetContext(qf_implicit, probl_ctx, sizeof *probl_ctx); - CeedQFunctionSetContext(qf_jacobian, probl_ctx, sizeof *probl_ctx); + CeedQFunctionContextCreate(ceed, &data->physCtx); + CeedQFunctionContextSetData(data->physCtx, CEED_MEM_HOST, CEED_USE_POINTER, + sizeof physCtxData, &physCtxData); + CeedQFunctionSetContext(qf_ics, data->physCtx); + CeedQFunctionContextCreate(ceed, &data->problCtx); + CeedQFunctionContextSetData(data->problCtx, CEED_MEM_HOST, CEED_USE_POINTER, + sizeof problCtxData, &problCtxData); + CeedQFunctionSetContext(qf_explicit, data->problCtx); + CeedQFunctionSetContext(qf_implicit, data->problCtx); + CeedQFunctionSetContext(qf_jacobian, data->problCtx); // Save libCEED data required for level // TODO: check how many of these are really needed outside data->basisx = basisx; diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index f0ad6489d5..58c4a00668 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -47,6 +47,7 @@ typedef struct { CeedScalar CtauS; CeedScalar strong_form; int stabilization; // See StabilizationType: 0=none, 1=SU, 2=SUPG + Mat T; } ProblemContext_s; typedef ProblemContext_s *ProblemContext; @@ -131,6 +132,7 @@ struct CeedData_ { CeedElemRestriction Erestrictx, Erestrictq, Erestrictqdi; CeedQFunction qf_setup, qf_mass, qf_ics, qf_explicit, qf_implicit, qf_jacobian; + CeedQFunctionContext physCtx, problCtx; CeedOperator op_setup, op_mass, op_ics, op_explicit, op_implicit, op_jacobian; CeedVector xcorners, xceed, qdata, q0ceed, mceed, hsceed, H0ceed; }; @@ -144,14 +146,20 @@ extern problemData problemOptions[]; // Auxiliary function to determine if nodes belong to cube face (panel) edges PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, - PetscInt ncomp, Mat *T); + PetscInt ncomp, PetscInt *edgenodecnt, + EdgeNode *edgenodes, Mat *T); -// Auxiliary function that sets up all corrdinate transformations between panels +// Auxiliary function that sets up all coordinate transformations between panels PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, PetscInt ncomp, EdgeNode edgenodes, PetscInt nedgenodes, Mat *T); +// Auxiliary function that converts global 3D coors into local panel coords +PetscErrorCode TransformCoords(DM dm, Vec Xloc, const PetscInt ncompx, + EdgeNode edgenodes, const PetscInt nedgenodes, + PhysicsContext phys_ctx, Vec *Xpanelsloc); + // ----------------------------------------------------------------------------- // Setup DM functions // ----------------------------------------------------------------------------- @@ -162,7 +170,8 @@ PetscErrorCode PetscFECreateByDegree(DM dm, PetscInt dim, PetscInt Nc, PetscInt order, PetscFE *fem); // Auxiliary function to setup DM FE space and info -PetscErrorCode SetupDM(DM dm, PetscInt degree, PetscInt ncompq, PetscInt dim); +PetscErrorCode SetupDMByDegree(DM dm, PetscInt degree, PetscInt ncompq, + PetscInt dim); // ----------------------------------------------------------------------------- // libCEED functions @@ -174,9 +183,10 @@ PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, CeedInt ncomp, // Auxiliary function to set up libCEED objects for a given degree PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, - PetscInt ncompx, PetscInt ncompq, User user, - CeedData data, problemData *problem, - PhysicsContext phys_ctx, ProblemContext probl_ctx); + const PetscInt ncompx, PetscInt ncompq, User user, + Vec Xloc, CeedData data, problemData *problem, + PhysicsContext physCtxData, + ProblemContext problCtxData); // ----------------------------------------------------------------------------- // RHS (Explicit part in time-stepper) function setup From a7d986f65fc04bd5f60a6c2207acba0deb6cbee8 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Mon, 17 Aug 2020 11:55:48 -0600 Subject: [PATCH 24/37] SWE: WIP geometric factors local to panel coord systems --- .../shallow-water/qfunctions/setup_geo.h | 169 +++++++----------- 1 file changed, 68 insertions(+), 101 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/setup_geo.h b/examples/fluids/shallow-water/qfunctions/setup_geo.h index 2befcb6e44..2ce7245757 100644 --- a/examples/fluids/shallow-water/qfunctions/setup_geo.h +++ b/examples/fluids/shallow-water/qfunctions/setup_geo.h @@ -77,125 +77,92 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, CeedScalar (*qdata)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // *INDENT-ON* + CeedInt panel = -1; + CeedPragmaSIMD - // Quadrature Point Loop to determine on which panel the element belongs to + // Quadrature point loop to determine which panel the element belongs to for (CeedInt i=0; i Date: Mon, 17 Aug 2020 16:56:52 -0600 Subject: [PATCH 25/37] SWE: Put back geometric factors as they were --- .../shallow-water/qfunctions/setup_geo.h | 218 +++++++++--------- 1 file changed, 115 insertions(+), 103 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/setup_geo.h b/examples/fluids/shallow-water/qfunctions/setup_geo.h index 2ce7245757..144791987e 100644 --- a/examples/fluids/shallow-water/qfunctions/setup_geo.h +++ b/examples/fluids/shallow-water/qfunctions/setup_geo.h @@ -26,44 +26,35 @@ // ***************************************************************************** // This QFunction sets up the geometric factors required for integration and -// coordinate transformations. See ref: "A Discontinuous Galerkin Transport -// Scheme on the Cubed Sphere", by Nair et al. (2004). +// coordinate transformations // -// Reference (parent) 2D coordinates: X \in [-1, 1]^2. +// Reference (parent) 2D coordinates: X \in [-1, 1]^2 // -// Local 2D physical coordinates on the 2D manifold: x \in [-l, l]^2 -// with l half edge of the cube inscribed in the sphere. These coordinate -// systems vary locally on each face (or panel) of the cube. +// Global 3D physical coordinates given by the mesh: xx \in [-R, R]^3 +// with R radius of the sphere // -// (theta, lambda) represnt latitude and longitude coordinates. +// Local 3D physical coordinates on the 2D manifold: x \in [-l, l]^3 +// with l half edge of the cube inscribed in the sphere // -// Change of coordinates from x (on the 2D manifold) to xx (phyisical 3D on -// the sphere), i.e., "cube-to-sphere" A, with equidistant central projection: +// Change of coordinates matrix computed by the library: +// (physical 3D coords relative to reference 2D coords) +// dxx_j/dX_i (indicial notation) [3 * 2] // -// For lateral panels (P0-P3): -// A = R cos(theta)cos(lambda) / l * [cos(lambda) 0] -// [-sin(theta)sin(lambda) cos(theta)] +// Change of coordinates x (on the 2D manifold) relative to xx (phyisical 3D): +// dx_i/dxx_j (indicial notation) [3 * 3] // -// For top panel (P4): -// A = R sin(theta) / l * [cos(lambda) sin(lambda)] -// [-sin(theta)sin(lambda) sin(theta)cos(lambda)] +// Change of coordinates x (on the 2D manifold) relative to X (reference 2D): +// (by chain rule) +// dx_i/dX_j [3 * 2] = dx_i/dxx_k [3 * 3] * dxx_k/dX_j [3 * 2] // -// For bottom panel(P5): -// A = R sin(theta) / l * [-cos(lambda) sin(lambda)] -// [sin(theta)sin(lambda) sin(theta)cos(lambda)] -// -// The inverse of A, A^{-1}, is the "sphere-to-cube" change of coordinates. -// -// The metric tensor g_{ij} = A^TA, and its inverse, -// g^{ij} = g_{ij}^{-1} = A^{-1}A^{-T} -// -// modJ represents the magnitude of the cross product of the columns of A, i.e., -// J = det(g_{ij}) +// modJ is given by the magnitude of the cross product of the columns of dx_i/dX_j // // The quadrature data is stored in the array qdata. // // We require the determinant of the Jacobian to properly compute integrals of -// the form: int( u v ). +// the form: int( u v ) +// +// Qdata: modJ * w // // ***************************************************************************** CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, @@ -77,92 +68,113 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, CeedScalar (*qdata)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // *INDENT-ON* - CeedInt panel = -1; - CeedPragmaSIMD - // Quadrature point loop to determine which panel the element belongs to + // Quadrature Point Loop for (CeedInt i=0; i Date: Tue, 18 Aug 2020 21:32:06 -0600 Subject: [PATCH 26/37] WIP SWE: pass sparse matrix to user context to be used in wrapper functions --- examples/fluids/shallow-water/shallowwater.c | 23 ++++++++---------- examples/fluids/shallow-water/src/setup.c | 23 ++++++++++++------ examples/fluids/shallow-water/src/wrappers.c | 25 ++++++++++++++++++++ examples/fluids/shallow-water/sw_headers.h | 5 ++-- 4 files changed, 53 insertions(+), 23 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 4a11f828fb..bfb661f720 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -59,11 +59,11 @@ int main(int argc, char **argv) { PetscInt degree, qextra, outputfreq, steps, contsteps, nedgenodes = 0; PetscMPIInt rank; PetscScalar ftime; - Vec Q, Qloc, Xloc, Xpanelsloc; + Vec Q, Qloc, Xloc; const PetscInt ncompx = 3; PetscInt viz_refine = 0; PetscBool read_mesh, simplex, test; - PetscInt topodim = 2, ncompq = 3, lnodes; + PetscInt topodim = 2, dim = 3, ncompq = 3, lnodes; // libCEED context char ceedresource[PETSC_MAX_PATH_LEN] = "/cpu/self", filename[PETSC_MAX_PATH_LEN]; @@ -196,7 +196,7 @@ int main(int argc, char **argv) { g *= mpersquareds; // Set up the libCEED context - PhysicsContext_s physCtxData = { + PhysicsContext_s physCtxData = { .u0 = 0., .v0 = 0., .h0 = .1, @@ -208,7 +208,7 @@ int main(int argc, char **argv) { .time = 0. }; - ProblemContext_s problCtxData = { + ProblemContext_s problCtxData = { .g = g, .H0 = H0, .CtauS = CtauS, @@ -378,14 +378,9 @@ int main(int argc, char **argv) { &T); CHKERRQ(ierr); - // Pass transformation matrix to context for setup operator - problCtxData.T = T; - - ierr = DMGetCoordinatesLocal(dm, &Xloc); CHKERRQ(ierr); - ierr = VecDuplicate(Xloc, &Xpanelsloc); CHKERRQ(ierr); // Transform coordinate according to their panel systems - ierr = TransformCoords(dm, Xloc, ncompx, edgenodes, nedgenodes, &physCtxData, - &Xpanelsloc); CHKERRQ(ierr); +// ierr = TransformCoords(dm, Xloc, ncompx, edgenodes, nedgenodes, &physCtxData, +// &Xpanelsloc); CHKERRQ(ierr); // Free edgenodes structure array ierr = PetscFree(edgenodes); CHKERRQ(ierr); @@ -393,7 +388,7 @@ int main(int argc, char **argv) { // Setup libCEED's objects ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, - Xpanelsloc, ceeddata, problem, &physCtxData, &problCtxData); + ceeddata, problem, &physCtxData, &problCtxData); CHKERRQ(ierr); // Set up PETSc context @@ -411,6 +406,7 @@ int main(int argc, char **argv) { user->dmviz = dmviz; user->interpviz = interpviz; user->ceed = ceed; + user->T = T; // Calculate qdata and ICs // Set up state global and local vectors @@ -421,7 +417,8 @@ int main(int argc, char **argv) { CeedQFunctionSetContext(ceeddata->qf_setup, ceeddata->physCtx); // Apply Setup Ceed Operators - ierr = VectorPlacePetscVec(ceeddata->xcorners, Xpanelsloc); CHKERRQ(ierr); + ierr = DMGetCoordinatesLocal(dm, &Xloc); CHKERRQ(ierr); + ierr = VectorPlacePetscVec(ceeddata->xcorners, Xloc); CHKERRQ(ierr); CeedOperatorApply(ceeddata->op_setup, ceeddata->xcorners, ceeddata->qdata, CEED_REQUEST_IMMEDIATE); ierr = ComputeLumpedMassMatrix(ceed, dm, ceeddata->Erestrictq, diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index d59087931e..0d542de45c 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -154,8 +154,8 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, (*edgenodecnt)++; } } - ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, *edgenodes, - *edgenodecnt, T); CHKERRQ(ierr); +// ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, *edgenodes, +// *edgenodecnt, T); CHKERRQ(ierr); // Free heap ierr = PetscFree(bitmap); CHKERRQ(ierr); @@ -494,7 +494,7 @@ PetscErrorCode PetscFECreateByDegree(DM dm, PetscInt dim, PetscInt Nc, // Auxiliary function to setup DM FE space and info // ----------------------------------------------------------------------------- -PetscErrorCode SetupDMByDegree(DM dm, PetscInt degree, PetscInt ncompq, +PetscErrorCode SetupDMByDegree(DM dm, PetscInt degree, PetscInt ncomp, PetscInt dim) { PetscErrorCode ierr; @@ -502,10 +502,10 @@ PetscErrorCode SetupDMByDegree(DM dm, PetscInt degree, PetscInt ncompq, { // Configure the finite element space PetscFE fe; - ierr = PetscFECreateByDegree(dm, dim, ncompq, PETSC_FALSE, NULL, degree, + ierr = PetscFECreateByDegree(dm, dim, ncomp, PETSC_FALSE, NULL, degree, &fe); ierr = PetscObjectSetName((PetscObject)fe, "Q"); CHKERRQ(ierr); - ierr = DMAddField(dm,NULL,(PetscObject)fe); CHKERRQ(ierr); + ierr = DMAddField(dm, NULL, (PetscObject)fe); CHKERRQ(ierr); ierr = DMCreateDS(dm); CHKERRQ(ierr); ierr = DMPlexSetClosurePermutationTensor(dm, PETSC_DETERMINE, NULL); @@ -652,11 +652,12 @@ PetscErrorCode VectorPlacePetscVec(CeedVector c, Vec p) { PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, PetscInt ncompx, PetscInt ncompq, User user, - Vec Xloc, CeedData data, problemData *problem, + CeedData data, problemData *problem, PhysicsContext physCtxData, ProblemContext problCtxData) { int ierr; DM dmcoord; + Vec Xloc; CeedBasis basisx, basisxc, basisq; CeedElemRestriction Erestrictx, Erestrictq, Erestrictqdi; CeedQFunction qf_setup, qf_ics, qf_explicit, qf_implicit, @@ -682,7 +683,8 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CHKERRQ(ierr); // CEED restrictions - ierr = CreateRestrictionPlex(ceed, dm, numP, ncompq, &Erestrictq); + // TODO: After the resitrction matrix is implemented comment the following line + ierr = CreateRestrictionPlex(ceed, dmcoord, numP, ncompq, &Erestrictq); CHKERRQ(ierr); ierr = CreateRestrictionPlex(ceed, dmcoord, 2, ncompx, &Erestrictx); CHKERRQ(ierr); @@ -692,6 +694,13 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CeedElemRestrictionCreateStrided(ceed, nelem, numQ*numQ, qdatasize, qdatasize*nelem*numQ*numQ, CEED_STRIDES_BACKEND, &Erestrictqdi); + // Solution vec restriction is a strided (identity) because we use a user + // mat mult before and after operator apply + // TODO: After the restriction matrix for the solution vector is implemented, + // decomment the following line +// CeedElemRestrictionCreateStrided(ceed, nelem, numP*numP, ncompq, +// ncompq*nelem*numP*numP, +// CEED_STRIDES_BACKEND, &Erestrictq); // Element coordinates ierr = DMGetCoordinatesLocal(dm, &Xloc); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/src/wrappers.c b/examples/fluids/shallow-water/src/wrappers.c index 8104bd10c4..d5a283dd48 100644 --- a/examples/fluids/shallow-water/src/wrappers.c +++ b/examples/fluids/shallow-water/src/wrappers.c @@ -46,10 +46,18 @@ PetscErrorCode FormRHSFunction_SW(TS ts, PetscReal t, Vec Q, Vec G, CeedVectorSetArray(user->qceed, CEED_MEM_HOST, CEED_USE_POINTER, q); CeedVectorSetArray(user->gceed, CEED_MEM_HOST, CEED_USE_POINTER, g); + // TODO: + // L-vector to E-vector: + // Apply user-defined restriction with appropriate coordinate transformations + // Apply CEED operator CeedOperatorApply(user->op_explicit, user->qceed, user->gceed, CEED_REQUEST_IMMEDIATE); + // TODO: + // E-vector to L-vector: + // Apply transpose of user-defined restriction + // Restore vectors ierr = VecRestoreArrayRead(Qloc, (const PetscScalar **)&q); CHKERRQ(ierr); ierr = VecRestoreArray(Gloc, &g); CHKERRQ(ierr); @@ -101,10 +109,19 @@ PetscErrorCode FormIFunction_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, (PetscScalar *)qdot); CeedVectorSetArray(user->fceed, CEED_MEM_HOST, CEED_USE_POINTER, f); + // TODO: + // L-vector to E-vector: + // Apply user-defined restriction with appropriate coordinate transformations + + // Apply CEED operator CeedOperatorApply(user->op_implicit, user->qceed, user->fceed, CEED_REQUEST_IMMEDIATE); + // TODO: + // E-vector to L-vector: + // Apply transpose of user-defined restriction + // Restore vectors ierr = VecRestoreArrayRead(Qloc, &q); CHKERRQ(ierr); ierr = VecRestoreArrayRead(Qdotloc, &qdot); CHKERRQ(ierr); @@ -179,11 +196,19 @@ PetscErrorCode ApplyJacobian_SW(Mat mat, Vec Q, Vec JVec) { CeedVectorSetArray(user->qceed, CEED_MEM_HOST, CEED_USE_POINTER, q); CeedVectorSetArray(user->jceed, CEED_MEM_HOST, CEED_USE_POINTER, j); + // TODO: + // L-vector to E-vector: + // Apply user-defined restriction with appropriate coordinate transformations + // Apply the CEED operator for the dF/dQ terms CeedOperatorApply(user->op_jacobian, user->qceed, user->jceed, CEED_REQUEST_IMMEDIATE); CeedVectorSyncArray(user->jceed, CEED_MEM_HOST); + // TODO: + // E-vector to L-vector: + // Apply transpose of user-defined restriction + // Restore PETSc vectors ierr = VecRestoreArrayRead(Qloc, (const PetscScalar **)&q); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index 58c4a00668..412fac62ee 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -47,7 +47,6 @@ typedef struct { CeedScalar CtauS; CeedScalar strong_form; int stabilization; // See StabilizationType: 0=none, 1=SU, 2=SUPG - Mat T; } ProblemContext_s; typedef ProblemContext_s *ProblemContext; @@ -101,7 +100,7 @@ struct User_ { PetscInt outputfreq; DM dm; DM dmviz; - Mat interpviz; + Mat interpviz, T; Ceed ceed; Units units; CeedVector qceed, q0ceed, qdotceed, fceed, gceed, jceed; @@ -184,7 +183,7 @@ PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, CeedInt ncomp, // Auxiliary function to set up libCEED objects for a given degree PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, const PetscInt ncompx, PetscInt ncompq, User user, - Vec Xloc, CeedData data, problemData *problem, + CeedData data, problemData *problem, PhysicsContext physCtxData, ProblemContext problCtxData); From a249dd26ed6edf5345998d632f1f22498e4f4edf Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 18 Aug 2020 22:02:48 -0600 Subject: [PATCH 27/37] SWE: Inside Setup QFunction make sure to project coordinate field onto unit sphere --- .../shallow-water/qfunctions/setup_geo.h | 18 ++++++++++++++---- examples/fluids/shallow-water/shallowwater.c | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/setup_geo.h b/examples/fluids/shallow-water/qfunctions/setup_geo.h index 144791987e..708515497a 100644 --- a/examples/fluids/shallow-water/qfunctions/setup_geo.h +++ b/examples/fluids/shallow-water/qfunctions/setup_geo.h @@ -20,6 +20,8 @@ #ifndef setupgeo_h #define setupgeo_h +#include "../sw_headers.h" + #ifndef __CUDACC__ # include #endif @@ -67,15 +69,23 @@ CEED_QFUNCTION(SetupGeo)(void *ctx, CeedInt Q, // Outputs CeedScalar (*qdata)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // *INDENT-ON* + // Context + const PhysicsContext context = (PhysicsContext)ctx; + const CeedScalar R = context->R; CeedPragmaSIMD // Quadrature Point Loop for (CeedInt i=0; i Date: Thu, 3 Sep 2020 15:38:50 -0600 Subject: [PATCH 28/37] examples/fluids: fix usage bugs --- examples/fluids/navier-stokes/Makefile | 2 +- examples/fluids/shallow-water/Makefile | 2 +- examples/fluids/shallow-water/src/setup.c | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/fluids/navier-stokes/Makefile b/examples/fluids/navier-stokes/Makefile index e820e47ab5..d642e912a8 100644 --- a/examples/fluids/navier-stokes/Makefile +++ b/examples/fluids/navier-stokes/Makefile @@ -18,7 +18,7 @@ COMMON ?= ../../common.mk -include $(COMMON) PETSc.pc := $(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig/PETSc.pc -CEED_DIR ?= ../.. +CEED_DIR ?= ../../.. ceed.pc := $(CEED_DIR)/lib/pkgconfig/ceed.pc CC = $(call pkgconf, --variable=ccompiler $(PETSc.pc) $(ceed.pc)) diff --git a/examples/fluids/shallow-water/Makefile b/examples/fluids/shallow-water/Makefile index c1a0b83b8a..796f4a1e84 100644 --- a/examples/fluids/shallow-water/Makefile +++ b/examples/fluids/shallow-water/Makefile @@ -15,7 +15,7 @@ # testbed platforms, in support of the nation's exascale computing imperative. PETSc.pc := $(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig/PETSc.pc -CEED_DIR ?= ../.. +CEED_DIR ?= ../../.. ceed.pc := $(CEED_DIR)/lib/pkgconfig/ceed.pc CC = $(call pkgconf, --variable=ccompiler $(PETSc.pc) $(ceed.pc)) diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 0d542de45c..b9fa6039f0 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -158,8 +158,7 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, // *edgenodecnt, T); CHKERRQ(ierr); // Free heap - ierr = PetscFree(bitmap); CHKERRQ(ierr); - ierr = PetscFree(bitmaploc); CHKERRQ(ierr); + ierr = PetscFree2(bitmaploc, bitmap); CHKERRQ(ierr); PetscFunctionReturn(0); } From dbc55b52963e9aa107b6238c401ebab7368fb327 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Fri, 4 Sep 2020 12:22:06 -0600 Subject: [PATCH 29/37] SWE: Fix set context bug after change in the interface --- examples/fluids/shallow-water/shallowwater.c | 13 +++++++++++-- examples/fluids/shallow-water/src/setup.c | 10 +--------- examples/fluids/shallow-water/sw_headers.h | 4 +--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index d640fa15e5..7378b3b585 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -385,10 +385,19 @@ int main(int argc, char **argv) { // Free edgenodes structure array ierr = PetscFree(edgenodes); CHKERRQ(ierr); - // Setup libCEED's objects + // Setup libCEED's objects and contexts ierr = PetscMalloc1(1, &ceeddata); CHKERRQ(ierr); + CeedQFunctionContextCreate(ceed, &ceeddata->physCtx); + CeedQFunctionContextSetData(ceeddata->physCtx, CEED_MEM_HOST, + CEED_USE_POINTER, sizeof physCtxData, + &physCtxData); + CeedQFunctionContextCreate(ceed, &ceeddata->problCtx); + CeedQFunctionContextSetData(ceeddata->problCtx, CEED_MEM_HOST, + CEED_USE_POINTER, sizeof problCtxData, + &problCtxData); + ierr = SetupLibceed(dm, ceed, degree, qextra, ncompx, ncompq, user, - ceeddata, problem, &physCtxData, &problCtxData); + ceeddata, problem); CHKERRQ(ierr); // Set up PETSc context diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index b9fa6039f0..4cee6d890e 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -651,9 +651,7 @@ PetscErrorCode VectorPlacePetscVec(CeedVector c, Vec p) { PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, PetscInt ncompx, PetscInt ncompq, User user, - CeedData data, problemData *problem, - PhysicsContext physCtxData, - ProblemContext problCtxData) { + CeedData data, problemData *problem) { int ierr; DM dmcoord; Vec Xloc; @@ -831,13 +829,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, user->op_jacobian = op_jacobian; // Set up the libCEED context - CeedQFunctionContextCreate(ceed, &data->physCtx); - CeedQFunctionContextSetData(data->physCtx, CEED_MEM_HOST, CEED_USE_POINTER, - sizeof physCtxData, &physCtxData); CeedQFunctionSetContext(qf_ics, data->physCtx); - CeedQFunctionContextCreate(ceed, &data->problCtx); - CeedQFunctionContextSetData(data->problCtx, CEED_MEM_HOST, CEED_USE_POINTER, - sizeof problCtxData, &problCtxData); CeedQFunctionSetContext(qf_explicit, data->problCtx); CeedQFunctionSetContext(qf_implicit, data->problCtx); CeedQFunctionSetContext(qf_jacobian, data->problCtx); diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index 412fac62ee..74e0831fb2 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -183,9 +183,7 @@ PetscErrorCode CreateRestrictionPlex(Ceed ceed, DM dm, CeedInt P, CeedInt ncomp, // Auxiliary function to set up libCEED objects for a given degree PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, const PetscInt ncompx, PetscInt ncompq, User user, - CeedData data, problemData *problem, - PhysicsContext physCtxData, - ProblemContext problCtxData); + CeedData data, problemData *problem); // ----------------------------------------------------------------------------- // RHS (Explicit part in time-stepper) function setup From f402779c363733ff1aaf1d59536f55f78419b375 Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 8 Sep 2020 09:51:10 -0600 Subject: [PATCH 30/37] SWE WIP: Fix a bug in advection QFunction Still need some debugging --- examples/fluids/shallow-water/qfunctions/advection.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fluids/shallow-water/qfunctions/advection.h b/examples/fluids/shallow-water/qfunctions/advection.h index 7b791f096c..6ba216fe6a 100644 --- a/examples/fluids/shallow-water/qfunctions/advection.h +++ b/examples/fluids/shallow-water/qfunctions/advection.h @@ -321,7 +321,7 @@ CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, u_dot_grad_h += u[j] * dhdx_j; } CeedScalar strongConv = (h + H0) *div_u + u_dot_grad_h; - CeedScalar strongResid = qdot[4][i] + strongConv; + CeedScalar strongResid = qdot[2][i] + strongConv; v[2][i] = wdetJ * qdot[2][i]; // transient part (ALWAYS) From ede22963086472851e70964b37f97748fa64a58d Mon Sep 17 00:00:00 2001 From: valeriabarra Date: Tue, 22 Sep 2020 20:26:14 -0600 Subject: [PATCH 31/37] SWE: Initial implementation of generic restriction matrix This generic restriction matrix does not only have 1's but also the coordinate transformations needed from the coord system of the nodes that are on the panel edges (lat, long) to the local ones of the nodes on the adjacent panels --- examples/fluids/shallow-water/shallowwater.c | 4 +- examples/fluids/shallow-water/src/setup.c | 270 +++++++++---------- examples/fluids/shallow-water/src/wrappers.c | 27 +- examples/fluids/shallow-water/sw_headers.h | 11 +- 4 files changed, 148 insertions(+), 164 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index 7378b3b585..c3e6c9db9f 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -374,8 +374,8 @@ int main(int argc, char **argv) { // Setup transformation matrix different panels of the cube Mat T; EdgeNode edgenodes; - ierr = FindPanelEdgeNodes(dm, &physCtxData, ncompq, &nedgenodes, &edgenodes, - &T); + ierr = FindPanelEdgeNodes(dm, &physCtxData, ncompq, degree, topodim, + &nedgenodes, &edgenodes, &T); CHKERRQ(ierr); // Transform coordinate according to their panel systems diff --git a/examples/fluids/shallow-water/src/setup.c b/examples/fluids/shallow-water/src/setup.c index 4cee6d890e..240d87f99e 100644 --- a/examples/fluids/shallow-water/src/setup.c +++ b/examples/fluids/shallow-water/src/setup.c @@ -76,7 +76,8 @@ problemData problemOptions[] = { // ----------------------------------------------------------------------------- PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, - PetscInt ncomp, PetscInt *edgenodecnt, + PetscInt ncomp, PetscInt degree, + PetscInt topodim, PetscInt *edgenodecnt, EdgeNode *edgenodes, Mat *T) { PetscInt ierr; @@ -154,8 +155,8 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, (*edgenodecnt)++; } } -// ierr = SetupPanelCoordTransformations(dm, phys_ctx, ncomp, *edgenodes, -// *edgenodecnt, T); CHKERRQ(ierr); + ierr = SetupRestrictionMatrix(dm, phys_ctx, degree, ncomp, *edgenodes, + *edgenodecnt, T); CHKERRQ(ierr); // Free heap ierr = PetscFree2(bitmaploc, bitmap); CHKERRQ(ierr); @@ -166,157 +167,147 @@ PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, // ----------------------------------------------------------------------------- // Auxiliary function that sets up all coordinate transformations between panels // ----------------------------------------------------------------------------- -PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, - PetscInt ncomp, - EdgeNode edgenodes, - PetscInt nedgenodes, Mat *T) { - PetscInt ierr; +PetscErrorCode SetupRestrictionMatrix(DM dm, PhysicsContext phys_ctx, + PetscInt degree, PetscInt ncomp, + EdgeNode edgenodes, PetscInt nedgenodes, + Mat *T) { + PetscInt ierr, nnodes, c, cStart, cEnd, nelem, numP, gdofs; MPI_Comm comm; Vec X; - PetscInt gdofs; + PetscSection section; const PetscScalar *xarray; PetscFunctionBeginUser; ierr = PetscObjectGetComm((PetscObject)dm, &comm); CHKERRQ(ierr); + ierr = DMGetSection(dm, §ion); CHKERRQ(ierr); ierr = DMGetCoordinates(dm, &X); CHKERRQ(ierr); ierr = VecGetSize(X, &gdofs); CHKERRQ(ierr); + nnodes = gdofs/ncomp; + ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr); + nelem = cEnd - cStart; + numP = degree + 1; // Preallocate sparse matrix - ierr = MatCreateBAIJ(comm, 2, PETSC_DECIDE, PETSC_DECIDE, 4*gdofs/ncomp, - 4*gdofs/ncomp, 2, NULL, 0, NULL, T); CHKERRQ(ierr); - for (PetscInt i=0; i < 4*gdofs/ncomp; i++) { - ierr = MatSetValue(*T, i, i, 1., INSERT_VALUES); CHKERRQ(ierr); - } + ierr = MatCreateAIJ(comm, nelem*numP*numP*ncomp, nnodes*ncomp, + PETSC_DETERMINE, PETSC_DETERMINE, 2, NULL, + nnodes*ncomp, NULL, T); CHKERRQ(ierr); - ierr = VecGetArrayRead(X, &xarray); CHKERRQ(ierr); - PetscScalar R = phys_ctx->R; - // Nodes loop - for (PetscInt i=0; i < gdofs; i += ncomp) { - // Read global Cartesian coordinates - PetscScalar x[3] = {xarray[i*ncomp + 0], - xarray[i*ncomp + 1], - xarray[i*ncomp + 2] - }; - // Normalize quadrature point coordinates to sphere - PetscScalar rad = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); - x[0] *= R / rad; - x[1] *= R / rad; - x[2] *= R / rad; - // Compute latitude and longitude - const PetscScalar theta = asin(x[2] / R); // latitude - const PetscScalar lambda = atan2(x[1], x[0]); // longitude - - // For P_0 (front), P_1 (east), P_2 (back), P_3 (west): - PetscScalar T00 = cos(theta)*cos(lambda) * cos(lambda); - PetscScalar T01 = cos(theta)*cos(lambda) * 0.; - PetscScalar T10 = cos(theta)*cos(lambda) * -sin(theta)*sin(lambda); - PetscScalar T11 = cos(theta)*cos(lambda) * cos(theta); - const PetscScalar T_lateral[2][2] = {{T00, - T01}, - {T10, - T11} - }; - PetscScalar Tinv00 = 1./(cos(theta)*cos(lambda)) * (1./cos(lambda)); - PetscScalar Tinv01 = 1./(cos(theta)*cos(lambda)) * 0.; - PetscScalar Tinv10 = 1./(cos(theta)*cos(lambda)) * tan(theta)*tan(lambda); - PetscScalar Tinv11 = 1./(cos(theta)*cos(lambda)) * (1./cos(theta)); - const PetscScalar T_lateralinv[2][2] = {{Tinv00, - Tinv01}, - {Tinv10, - Tinv11} - }; - // For P4 (north): - T00 = sin(theta) * cos(lambda); - T01 = sin(theta) * sin(lambda); - T10 = sin(theta) * -sin(theta)*sin(lambda); - T11 = sin(theta) * sin(theta)*cos(lambda); - const PetscScalar T_top[2][2] = {{T00, - T01}, - {T10, - T11} - }; - Tinv00 = 1./(sin(theta)*sin(theta)) * sin(theta)*cos(lambda); - Tinv01 = 1./(sin(theta)*sin(theta)) * (-sin(lambda)); - Tinv10 = 1./(sin(theta)*sin(theta)) * sin(theta)*sin(lambda); - Tinv11 = 1./(sin(theta)*sin(theta)) * cos(lambda); - const PetscScalar T_topinv[2][2] = {{Tinv00, - Tinv01}, - {Tinv10, - Tinv11} - }; - - // For P5 (south): - T00 = sin(theta) * (-cos(theta)); - T01 = sin(theta) * sin(lambda); - T10 = sin(theta) * sin(theta)*sin(lambda); - T11 = sin(theta) * sin(theta)*cos(lambda); - const PetscScalar T_bottom[2][2] = {{T00, - T01}, - {T10, - T11} - }; - Tinv00 = 1./(sin(theta)*sin(theta)) * (-sin(theta)*cos(lambda)); - Tinv01 = 1./(sin(theta)*sin(theta)) * sin(lambda); - Tinv10 = 1./(sin(theta)*sin(theta)) * sin(theta)*sin(lambda); - Tinv11 = 1./(sin(theta)*sin(theta)) * cos(lambda); - const PetscScalar T_bottominv[2][2] = {{Tinv00, - Tinv01}, - {Tinv10, - Tinv11} - }; - - const PetscScalar (*transforms[6])[2][2] = {&T_lateral, - &T_lateral, - &T_lateral, - &T_lateral, - &T_top, - &T_bottom - }; - - const PetscScalar (*inv_transforms[6])[2][2] = {&T_lateralinv, - &T_lateralinv, - &T_lateralinv, - &T_lateralinv, - &T_topinv, - &T_bottominv - }; + // Loop over elements + for (c = cStart; c < cEnd; c++) { + PetscInt numindices, *indices, i; + ierr = DMPlexGetClosureIndices(dm, section, section, c, PETSC_TRUE, + &numindices, &indices, NULL, NULL); + CHKERRQ(ierr); + for (i = 0; i < numindices; i += ncomp) { + for (PetscInt j = 0; j < ncomp; j++) { + if (indices[i+j] != indices[i] + (PetscInt)(copysign(j, indices[i]))) + SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, + "Cell %D closure indices not interlaced", c); + } + // NO BC on closed surfaces + PetscInt loc = indices[i]; - for (PetscInt e = 0; e < nedgenodes; e++) { - if (edgenodes[e].idx == i) { - const PetscScalar (*matrixA)[2][2] = inv_transforms[edgenodes[e].panelA]; - const PetscScalar (*matrixB)[2][2] = transforms[edgenodes[e].panelB]; - - // inv_transform * transform (A^{-1}*B) - // This product represents the mapping from coordinate system A - // to spherical coordinates and then to coordinate system B. Vice versa - // for transform * inv_transform (B^{-1}*A) - PetscScalar matrixAB[2][2], matrixBA[2][2]; - for (int j=0; j<2; j++) { - for (int k=0; k<2; k++) { - matrixAB[j][k] = matrixBA[j][k] = 0; - for (int l=0; l<2; l++) { - matrixAB[j][k] += (*matrixA)[j][l] * (*matrixB)[l][k]; - matrixBA[j][k] += (*matrixB)[j][l] * (*matrixA)[l][k]; - } - } + // Query element panel + PetscInt panel; + ierr = DMGetLabelValue(dm, "panel", c, &panel); CHKERRQ(ierr); + + // Query if edge node + PetscBool isEdgeNode = PETSC_FALSE; + + for (PetscInt e = 0; e < nedgenodes; e++) { + if (edgenodes[e].idx == loc) { + isEdgeNode = PETSC_TRUE; + break; } - PetscInt idxAB[2] = {4*i/ncomp + 0, 4*i/ncomp +1}; - PetscInt idxBA[2] = {4*i/ncomp + 2, 4*i/ncomp +3}; - ierr = MatSetValues(*T, 2, idxAB, 2, idxAB, (PetscScalar *)matrixAB, - INSERT_VALUES); CHKERRQ(ierr); - ierr = MatSetValues(*T, 2, idxBA, 2, idxBA, (PetscScalar *)matrixBA, - INSERT_VALUES); CHKERRQ(ierr); } + + PetscScalar entries[9]; + if (isEdgeNode) { + ierr = VecGetArrayRead(X, &xarray); CHKERRQ(ierr); + PetscScalar R = phys_ctx->R; + // Read global Cartesian coordinates + PetscScalar x[3] = {xarray[loc + 0], + xarray[loc + 1], + xarray[loc + 2] + }; + // Restore array read + ierr = VecRestoreArrayRead(X, &xarray); CHKERRQ(ierr); + + // Normalize quadrature point coordinates to sphere + PetscScalar rad = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); + x[0] *= R / rad; + x[1] *= R / rad; + x[2] *= R / rad; + // Compute latitude and longitude + const PetscScalar theta = asin(x[2] / R); // latitude + const PetscScalar lambda = atan2(x[1], x[0]); // longitude + + PetscScalar T00, T01, T10, T11; + + switch (panel) { + case 0: + case 1: + case 2: + case 3: + // For P_0 (front), P_1 (east), P_2 (back), P_3 (west): + T00 = cos(theta)*cos(lambda) * cos(lambda); + T01 = cos(theta)*cos(lambda) * 0.; + T10 = cos(theta)*cos(lambda) * -sin(theta)*sin(lambda); + T11 = cos(theta)*cos(lambda) * cos(theta); + break; + case 4: + // For P4 (north): + T00 = sin(theta) * cos(lambda); + T01 = sin(theta) * sin(lambda); + T10 = sin(theta) * -sin(theta)*sin(lambda); + T11 = sin(theta) * sin(theta)*cos(lambda); + break; + case 5: + // For P5 (south): + T00 = sin(theta) * (-cos(theta)); + T01 = sin(theta) * sin(lambda); + T10 = sin(theta) * sin(theta)*sin(lambda); + T11 = sin(theta) * sin(theta)*cos(lambda); + break; + } + + entries[0] = T00; + entries[1] = T01; + entries[2] = 0.; + entries[3] = T10; + entries[4] = T11; + entries[5] = 0.; + entries[6] = 0.; + entries[7] = 0.; + entries[8] = 1.; + + } else { + entries[0] = 1.; + entries[1] = 0.; + entries[2] = 0.; + entries[3] = 0.; + entries[4] = 1.; + entries[5] = 0.; + entries[6] = 0.; + entries[7] = 0.; + entries[8] = 1.; + } + PetscInt elem = c - cStart; + PetscInt idxrow[3] = {elem*loc + 0, elem*loc + 1, elem*loc + 2}; + PetscInt idxcol[3] = {loc + 0, loc + 1, loc + 2}; + ierr = MatSetValues(*T, ncomp, idxrow, ncomp, idxcol, entries, + INSERT_VALUES); CHKERRQ(ierr); } + ierr = DMPlexRestoreClosureIndices(dm, section, section, c, PETSC_TRUE, + &numindices, &indices, NULL, NULL); + CHKERRQ(ierr); } + // Assemble matrix for all node transformations ierr = MatAssemblyBegin(*T, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = MatAssemblyEnd(*T, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - // Restore array read - ierr = VecRestoreArrayRead(X, &xarray); CHKERRQ(ierr); - PetscFunctionReturn(0); } @@ -377,9 +368,7 @@ PetscErrorCode TransformCoords(DM dm, Vec Xloc, const PetscInt ncompx, xpanelslocarray[n*ncompx+1] = lambda; xpanelslocarray[n*ncompx+2] = -1; } - else { - switch (panel) { case 0: x = l * (Y / X); @@ -680,9 +669,6 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CHKERRQ(ierr); // CEED restrictions - // TODO: After the resitrction matrix is implemented comment the following line - ierr = CreateRestrictionPlex(ceed, dmcoord, numP, ncompq, &Erestrictq); - CHKERRQ(ierr); ierr = CreateRestrictionPlex(ceed, dmcoord, 2, ncompx, &Erestrictx); CHKERRQ(ierr); @@ -693,11 +679,9 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, CeedInt degree, CeedInt qextra, CEED_STRIDES_BACKEND, &Erestrictqdi); // Solution vec restriction is a strided (identity) because we use a user // mat mult before and after operator apply - // TODO: After the restriction matrix for the solution vector is implemented, - // decomment the following line -// CeedElemRestrictionCreateStrided(ceed, nelem, numP*numP, ncompq, -// ncompq*nelem*numP*numP, -// CEED_STRIDES_BACKEND, &Erestrictq); + CeedElemRestrictionCreateStrided(ceed, nelem, numP*numP, ncompq, + ncompq*nelem*numP*numP, + CEED_STRIDES_BACKEND, &Erestrictq); // Element coordinates ierr = DMGetCoordinatesLocal(dm, &Xloc); CHKERRQ(ierr); diff --git a/examples/fluids/shallow-water/src/wrappers.c b/examples/fluids/shallow-water/src/wrappers.c index d5a283dd48..9ce092b479 100644 --- a/examples/fluids/shallow-water/src/wrappers.c +++ b/examples/fluids/shallow-water/src/wrappers.c @@ -40,16 +40,16 @@ PetscErrorCode FormRHSFunction_SW(TS ts, PetscReal t, Vec Q, Vec G, ierr = DMGlobalToLocal(user->dm, Q, INSERT_VALUES, Qloc); CHKERRQ(ierr); ierr = VecZeroEntries(Gloc); CHKERRQ(ierr); + // TODO: + // L-vector to E-vector: + // Apply user-defined restriction with appropriate coordinate transformations to Qloc + // Ceed Vectors ierr = VecGetArrayRead(Qloc, (const PetscScalar **)&q); CHKERRQ(ierr); ierr = VecGetArray(Gloc, &g); CHKERRQ(ierr); CeedVectorSetArray(user->qceed, CEED_MEM_HOST, CEED_USE_POINTER, q); CeedVectorSetArray(user->gceed, CEED_MEM_HOST, CEED_USE_POINTER, g); - // TODO: - // L-vector to E-vector: - // Apply user-defined restriction with appropriate coordinate transformations - // Apply CEED operator CeedOperatorApply(user->op_explicit, user->qceed, user->gceed, CEED_REQUEST_IMMEDIATE); @@ -99,6 +99,10 @@ PetscErrorCode FormIFunction_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, ierr = DMGlobalToLocal(user->dm, Qdot, INSERT_VALUES, Qdotloc); CHKERRQ(ierr); ierr = VecZeroEntries(Floc); CHKERRQ(ierr); + // TODO: + // L-vector to E-vector: + // Apply user-defined restriction with appropriate coordinate transformations to Qloc + // Ceed Vectors ierr = VecGetArrayRead(Qloc, &q); CHKERRQ(ierr); ierr = VecGetArrayRead(Qdotloc, &qdot); CHKERRQ(ierr); @@ -109,11 +113,6 @@ PetscErrorCode FormIFunction_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, (PetscScalar *)qdot); CeedVectorSetArray(user->fceed, CEED_MEM_HOST, CEED_USE_POINTER, f); - // TODO: - // L-vector to E-vector: - // Apply user-defined restriction with appropriate coordinate transformations - - // Apply CEED operator CeedOperatorApply(user->op_implicit, user->qceed, user->fceed, CEED_REQUEST_IMMEDIATE); @@ -190,16 +189,16 @@ PetscErrorCode ApplyJacobian_SW(Mat mat, Vec Q, Vec JVec) { ierr = DMGlobalToLocal(user->dm, Q, INSERT_VALUES, Qloc); CHKERRQ(ierr); ierr = VecZeroEntries(Jloc); CHKERRQ(ierr); - // Setup CEED vectors + // TODO: + // L-vector to E-vector: + // Apply user-defined restriction with appropriate coordinate transformations to Qloc + + // CEED vectors ierr = VecGetArrayRead(Qloc, (const PetscScalar **)&q); CHKERRQ(ierr); ierr = VecGetArray(Jloc, &j); CHKERRQ(ierr); CeedVectorSetArray(user->qceed, CEED_MEM_HOST, CEED_USE_POINTER, q); CeedVectorSetArray(user->jceed, CEED_MEM_HOST, CEED_USE_POINTER, j); - // TODO: - // L-vector to E-vector: - // Apply user-defined restriction with appropriate coordinate transformations - // Apply the CEED operator for the dF/dQ terms CeedOperatorApply(user->op_jacobian, user->qceed, user->jceed, CEED_REQUEST_IMMEDIATE); diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index 74e0831fb2..1a92351fdd 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -145,14 +145,15 @@ extern problemData problemOptions[]; // Auxiliary function to determine if nodes belong to cube face (panel) edges PetscErrorCode FindPanelEdgeNodes(DM dm, PhysicsContext phys_ctx, - PetscInt ncomp, PetscInt *edgenodecnt, + PetscInt ncomp, PetscInt degree, + PetscInt topodim, PetscInt *edgenodecnt, EdgeNode *edgenodes, Mat *T); // Auxiliary function that sets up all coordinate transformations between panels -PetscErrorCode SetupPanelCoordTransformations(DM dm, PhysicsContext phys_ctx, - PetscInt ncomp, - EdgeNode edgenodes, - PetscInt nedgenodes, Mat *T); +PetscErrorCode SetupRestrictionMatrix(DM dm, PhysicsContext phys_ctx, + PetscInt degree, PetscInt ncomp, + EdgeNode edgenodes, PetscInt nedgenodes, + Mat *T); // Auxiliary function that converts global 3D coors into local panel coords PetscErrorCode TransformCoords(DM dm, Vec Xloc, const PetscInt ncompx, From e1808a2c9fb46a1831918eef66483bb23ac89384 Mon Sep 17 00:00:00 2001 From: Leila Ghaffari Date: Tue, 9 Mar 2021 15:06:50 -0700 Subject: [PATCH 32/37] examples/fluids: Move setup-boundary.h to navier-stokes --- examples/fluids/{ => navier-stokes}/setup-boundary.h | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/fluids/{ => navier-stokes}/setup-boundary.h (100%) diff --git a/examples/fluids/setup-boundary.h b/examples/fluids/navier-stokes/setup-boundary.h similarity index 100% rename from examples/fluids/setup-boundary.h rename to examples/fluids/navier-stokes/setup-boundary.h From 568e5c5b0d112f2834227be84a3a82259401f031 Mon Sep 17 00:00:00 2001 From: Leila Ghaffari Date: Tue, 9 Mar 2021 15:15:07 -0700 Subject: [PATCH 33/37] SWE: fixed the double prototyping --- examples/fluids/shallow-water/sw_headers.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index 1a92351fdd..c9c38097b4 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -202,10 +202,6 @@ PetscErrorCode FormRHSFunction_SW(TS ts, PetscReal t, Vec Q, Vec G, PetscErrorCode FormIFunction_SW(TS ts, PetscReal t, Vec Q, Vec Qdot, Vec F, void *userData); -// This forms the RHS of the IMEX ODE, given as F(t,Q,Q_t) = G(t,Q) -PetscErrorCode FormRHSFunction_SW(TS ts, PetscReal t, Vec Q, Vec G, - void *userData); - // ----------------------------------------------------------------------------- // Jacobian setup and apply // ----------------------------------------------------------------------------- From af0adafa2d315904f367216d2fd6640cfbdb5c14 Mon Sep 17 00:00:00 2001 From: Leila Ghaffari Date: Tue, 9 Mar 2021 15:20:44 -0700 Subject: [PATCH 34/37] SWE: style --- examples/fluids/shallow-water/shallowwater.c | 17 +++++++++++------ examples/fluids/shallow-water/sw_headers.h | 10 +++++----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/fluids/shallow-water/shallowwater.c b/examples/fluids/shallow-water/shallowwater.c index c3e6c9db9f..0b9a384316 100644 --- a/examples/fluids/shallow-water/shallowwater.c +++ b/examples/fluids/shallow-water/shallowwater.c @@ -38,7 +38,8 @@ /// @file /// Shallow-water equations example using PETSc -const char help[] = "Solve the shallow-water equations using PETSc and libCEED\n"; +const char help[] = + "Solve the shallow-water equations using PETSc and libCEED\n"; #include "sw_headers.h" @@ -78,7 +79,8 @@ int main(int argc, char **argv) { PetscScalar R_e = 6.37122e6; // Earth radius (m) PetscScalar g = 9.81; // gravitational acceleration (m/s^2) PetscScalar H0 = 0; // constant mean height (m) - PetscScalar gamma = 0; // angle between axis of rotation and polar axis + PetscScalar gamma = + 0; // angle between axis of rotation and polar axis PetscScalar mpersquareds; // Check PETSc CUDA support PetscBool petschavecuda, setmemtyperequest = PETSC_FALSE; @@ -102,7 +104,8 @@ int main(int argc, char **argv) { // Parse command line options comm = PETSC_COMM_WORLD; - ierr = PetscOptionsBegin(comm, NULL, "Shallow-water equations in PETSc with libCEED", + ierr = PetscOptionsBegin(comm, NULL, + "Shallow-water equations in PETSc with libCEED", NULL); CHKERRQ(ierr); ierr = PetscOptionsString("-ceed", "CEED resource specifier", NULL, ceedresource, ceedresource, @@ -152,7 +155,8 @@ int main(int argc, char **argv) { NULL, second, &second, NULL); CHKERRQ(ierr); second = fabs(second); outputfreq = 10; - ierr = PetscOptionsInt("-output_freq", "Frequency of output, in number of steps", + ierr = PetscOptionsInt("-output_freq", + "Frequency of output, in number of steps", NULL, outputfreq, &outputfreq, NULL); CHKERRQ(ierr); contsteps = 0; ierr = PetscOptionsInt("-continue", "Continue from previous solution", @@ -167,7 +171,8 @@ int main(int argc, char **argv) { NULL, g, &g, NULL); CHKERRQ(ierr); ierr = PetscOptionsScalar("-H0", "Mean height", NULL, H0, &H0, NULL); CHKERRQ(ierr); - ierr = PetscOptionsScalar("-H0", "Angle between axis of rotation and polar axis", + ierr = PetscOptionsScalar("-H0", + "Angle between axis of rotation and polar axis", NULL, gamma, &gamma, NULL); CHKERRQ(ierr); PetscStrncpy(user->outputfolder, ".", 2); ierr = PetscOptionsString("-of", "Output folder", @@ -563,7 +568,7 @@ int main(int argc, char **argv) { ierr = DMDestroy(&dmviz); CHKERRQ(ierr); ierr = TSDestroy(&ts); CHKERRQ(ierr); ierr = DMDestroy(&dm); CHKERRQ(ierr); - ierr = MatDestroy(&J);CHKERRQ(ierr); + ierr = MatDestroy(&J); CHKERRQ(ierr); ierr = PetscFree(units); CHKERRQ(ierr); ierr = PetscFree(user); CHKERRQ(ierr); return PetscFinalize(); diff --git a/examples/fluids/shallow-water/sw_headers.h b/examples/fluids/shallow-water/sw_headers.h index c9c38097b4..536047a959 100644 --- a/examples/fluids/shallow-water/sw_headers.h +++ b/examples/fluids/shallow-water/sw_headers.h @@ -56,7 +56,7 @@ typedef struct { CeedQFunctionUser setup, ics, apply_explfunction, apply_implfunction, apply_jacobian; const char *setup_loc, *ics_loc, *apply_explfunction_loc, - *apply_implfunction_loc, *apply_jacobian_loc; + *apply_implfunction_loc, *apply_jacobian_loc; const bool non_zero_time; } problemData; @@ -235,12 +235,12 @@ PetscErrorCode VectorPlacePetscVec(CeedVector c, Vec p); // Auxiliary function to apply the ICs and eliminate repeated values in initial // state vector, arising from restriction PetscErrorCode ICs_FixMultiplicity(CeedOperator op_ics, - CeedVector xcorners, CeedVector q0ceed, DM dm, Vec Qloc, Vec Q, - CeedElemRestriction restrictq, PhysicsContext ctx, CeedScalar time); + CeedVector xcorners, CeedVector q0ceed, DM dm, Vec Qloc, Vec Q, + CeedElemRestriction restrictq, PhysicsContext ctx, CeedScalar time); // Auxiliary function to compute the lumped mass matrix PetscErrorCode ComputeLumpedMassMatrix(Ceed ceed, DM dm, - CeedElemRestriction restrictq, CeedBasis basisq, - CeedElemRestriction restrictqdi, CeedVector qdata, Vec M); + CeedElemRestriction restrictq, CeedBasis basisq, + CeedElemRestriction restrictqdi, CeedVector qdata, Vec M); #endif // sw_headers_h From af734c2e57e6dd87373b68324ce93d93525b8305 Mon Sep 17 00:00:00 2001 From: Leila Ghaffari Date: Mon, 15 Mar 2021 16:21:08 -0600 Subject: [PATCH 35/37] SWE: fix the weak form in the explicit scheme --- examples/fluids/shallow-water/qfunctions/geostrophic.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/geostrophic.h b/examples/fluids/shallow-water/qfunctions/geostrophic.h index a2fa2bd33d..2864e80e07 100644 --- a/examples/fluids/shallow-water/qfunctions/geostrophic.h +++ b/examples/fluids/shallow-water/qfunctions/geostrophic.h @@ -154,7 +154,9 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Explicit spatial terms of G_1(t,q): // Explicit terms multiplying v // - (omega + f) * khat curl u - grad(|u|^2/2) // TODO: needs fix with weak form - v[0][i] = - wdetJ*(u[0]*du[0][0] + u[1]*du[0][1] + f*u[1]); + // -- grad(|u|^2/2) = u^T grad u + // -- khat curl u = [-u_1, u_0] + v[0][i] = - wdetJ*(u[0]*du[0][0] + u[1]*du[1][0] - f*u[1]); // No explicit terms multiplying dv dv[0][0][i] = 0; dv[1][0][i] = 0; @@ -162,7 +164,9 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Explicit spatial terms of G_2(t,q): // Explicit terms multiplying v // - (omega + f) * khat curl u - grad(|u|^2/2) // TODO: needs fix with weak form - v[1][i] = - wdetJ*(u[0]*du[1][0] + u[1]*du[1][1] - f*u[0]); + // -- grad(|u|^2/2) = u^T grad u + // -- khat curl u = [-u_1, u_0] + v[1][i] = - wdetJ*(u[0]*du[0][1] + u[1]*du[1][1] + f*u[0]); // No explicit terms multiplying dv dv[0][1][i] = 0; dv[1][1][i] = 0; From 3c01fc9129e448694af0998cad4da44fa2c68226 Mon Sep 17 00:00:00 2001 From: Leila Ghaffari Date: Tue, 16 Mar 2021 14:44:04 -0600 Subject: [PATCH 36/37] SWE: more comments and update the docs --- examples/fluids/index.rst | 8 +++++--- .../fluids/shallow-water/qfunctions/geostrophic.h | 14 ++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/fluids/index.rst b/examples/fluids/index.rst index 828f9b2333..9bb434cce0 100644 --- a/examples/fluids/index.rst +++ b/examples/fluids/index.rst @@ -377,7 +377,9 @@ where the time derivative :math:`\bm{\dot q}` is defined by \bm{\dot q}(\bm q) = \sigma \bm q + \bm z in terms of :math:`\bm z` from prior state and :math:`\sigma > 0`, -both of which depend on the specific time integration scheme. We split the implicit and explicit terms as follows: +both of which depend on the specific time integration scheme. We split the time +integration scheme into implicit and explicit parts where the implicit terms is +given in what follows. .. math:: :label: eq-swe-implicit-part @@ -399,8 +401,8 @@ While for the explicit part we specify \bm G(t, \bm q) := \left\{ \begin{array}{l} - G_1 (u_{\lambda}, u_{\theta}, h) = - u_{\lambda} \frac{\partial u_{\lambda}}{\partial \alpha} - u_{\theta} \frac{\partial u_{\lambda}}{\partial \beta} - f u_{\theta}\\ - G_2 (u_{\lambda}, u_{\theta}, h) = - u_{\lambda} \frac{\partial u_{\theta}}{\partial \alpha} - u_{\theta} \frac{\partial u_{\theta}}{\partial \beta} + f u_{\lambda}\\ + G_1 (u_{\lambda}, u_{\theta}, h) = - u_{\lambda} \frac{\partial u_{\lambda}}{\partial \alpha} - u_{\theta} \frac{\partial u_{\theta}}{\partial \alpha} + f u_{\theta}\\ + G_2 (u_{\lambda}, u_{\theta}, h) = - u_{\lambda} \frac{\partial u_{\lambda}}{\partial \beta} - u_{\theta} \frac{\partial u_{\theta}}{\partial \beta} - f u_{\lambda}\\ G_3 (u_{\lambda}, u_{\theta}, h) = 0 . \end{array} \right. diff --git a/examples/fluids/shallow-water/qfunctions/geostrophic.h b/examples/fluids/shallow-water/qfunctions/geostrophic.h index 2864e80e07..189fe60c59 100644 --- a/examples/fluids/shallow-water/qfunctions/geostrophic.h +++ b/examples/fluids/shallow-water/qfunctions/geostrophic.h @@ -153,9 +153,10 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // The Physics // Explicit spatial terms of G_1(t,q): // Explicit terms multiplying v - // - (omega + f) * khat curl u - grad(|u|^2/2) // TODO: needs fix with weak form - // -- grad(|u|^2/2) = u^T grad u - // -- khat curl u = [-u_1, u_0] + // - (omega + f) * khat curl u - grad(|u|^2/2) + // -- omega khat curl u = 0 (u_3 = 0) + // -- f khat curl u = f [-u_1, u_0] + // -- grad(|u|^2/2) = (grad u)^T u v[0][i] = - wdetJ*(u[0]*du[0][0] + u[1]*du[1][0] - f*u[1]); // No explicit terms multiplying dv dv[0][0][i] = 0; @@ -163,9 +164,10 @@ CEED_QFUNCTION(SWExplicit)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Explicit spatial terms of G_2(t,q): // Explicit terms multiplying v - // - (omega + f) * khat curl u - grad(|u|^2/2) // TODO: needs fix with weak form - // -- grad(|u|^2/2) = u^T grad u - // -- khat curl u = [-u_1, u_0] + // - (omega + f) * khat curl u - grad(|u|^2/2) + // -- omega khat curl u = 0 (u_3 = 0) + // -- f khat curl u = f [-u_1, u_0] + // -- grad(|u|^2/2) = (grad u)^T u v[1][i] = - wdetJ*(u[0]*du[0][1] + u[1]*du[1][1] + f*u[0]); // No explicit terms multiplying dv dv[0][1][i] = 0; From da12e016b046f7681b55779c4ab9ab6734637994 Mon Sep 17 00:00:00 2001 From: Leila Ghaffari Date: Wed, 17 Mar 2021 11:51:23 -0600 Subject: [PATCH 37/37] SWE: some comments on the div operations --- .../shallow-water/qfunctions/advection.h | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/fluids/shallow-water/qfunctions/advection.h b/examples/fluids/shallow-water/qfunctions/advection.h index 6ba216fe6a..b12daa4c98 100644 --- a/examples/fluids/shallow-water/qfunctions/advection.h +++ b/examples/fluids/shallow-water/qfunctions/advection.h @@ -185,13 +185,13 @@ CEED_QFUNCTION(SWExplicit_Advection)(void *ctx, CeedInt Q, // -- Height: // Evaluate the strong form using // div((h + H0) u) = u . grad(h) + (h + H0) div(u), with H0 constant - // or in index notation: (u_j h)_{,j} = u_j h_j + (h + H0) u_{j,j} + // or in index notation: (u_j h)_{,j} = u_j h_{,j} + (h + H0) u_{j,j} CeedScalar div_u = 0, u_dot_grad_h = 0; - for (CeedInt j=0; j<2; j++) { + for (CeedInt j=0; j<2; j++) { // we skip j=2 since dqdx[2] is zero CeedScalar dhdx_j = 0; - for (CeedInt k=0; k<2; k++) { // TODO: check indices! in this case dXdx is 2x3 + for (CeedInt k=0; k<2; k++) { div_u += du[j][k] * dXdx[k][j]; // u_{j,j} = u_{j,K} X_{K,j} - dhdx_j += dh[k] * dXdx[k][j]; + dhdx_j += dh[k] * dXdx[k][j]; // h_{,j} = h_K X_{K,j} } u_dot_grad_h += u[j] * dhdx_j; } @@ -310,13 +310,13 @@ CEED_QFUNCTION(SWImplicit_Advection)(void *ctx, CeedInt Q, // -- Height: // Evaluate the strong form using // div((h + H0) u) = u . grad(h) + (h + H0) div(u), with H0 constant - // or in index notation: (u_j h)_{,j} = u_j h_j + (h + H0) u_{j,j} + // or in index notation: (u_j h)_{,j} = u_j h_{,j} + (h + H0) u_{j,j} CeedScalar div_u = 0, u_dot_grad_h = 0; - for (CeedInt j=0; j<2; j++) { + for (CeedInt j=0; j<2; j++) { // we skip j=2 since dqdx[2] is zero CeedScalar dhdx_j = 0; - for (CeedInt k=0; k<2; k++) { // TODO: check indices! in this case dXdx is 2x3 + for (CeedInt k=0; k<2; k++) { div_u += du[j][k] * dXdx[k][j]; // u_{j,j} = u_{j,K} X_{K,j} - dhdx_j += dh[k] * dXdx[k][j]; + dhdx_j += dh[k] * dXdx[k][j]; // h_{,j} = h_K X_{K,j} } u_dot_grad_h += u[j] * dhdx_j; } @@ -461,13 +461,13 @@ CEED_QFUNCTION(SWJacobian_Advection)(void *ctx, CeedInt Q, // -- Height: // Evaluate the strong form of the action of the Jacobian using // div((h + H0) delta u) = delta u . grad(h) + (h + H0) div(delta u), with H0 constant - // or in index notation: (delta u_j h)_{,j} = delta u_j h_j + (h + H0) delta u_{j,j} + // or in index notation: (delta u_j h)_{,j} = delta u_j h_{,j} + (h + H0) delta u_{j,j} CeedScalar div_deltau = 0, deltau_dot_grad_h = 0; - for (CeedInt j=0; j<2; j++) { + for (CeedInt j=0; j<2; j++) { // we skip j=2 since dqdx[2] is zero CeedScalar dhdx_j = 0; - for (CeedInt k=0; k<2; k++) { // TODO: check indices! in this case dXdx is 2x3 - div_deltau += deltadu[j][k] * dXdx[k][j]; // u_{j,j} = u_{j,K} X_{K,j} - dhdx_j += dh[k] * dXdx[k][j]; + for (CeedInt k=0; k<2; k++) { + div_deltau += deltadu[j][k] * dXdx[k][j]; // deltau_{j,j} = deltau_{j,K} X_{K,j} + dhdx_j += dh[k] * dXdx[k][j]; // h_{,j} = h_K X_{K,j} } deltau_dot_grad_h += deltau[j] * dhdx_j; }