From 39b29e6bb02d5e712a48a9e332947acc8ac654f0 Mon Sep 17 00:00:00 2001 From: Daniel Schwen Date: Tue, 21 Jan 2020 18:16:11 -0700 Subject: [PATCH] Demonstrate working transform (#401) --- examples/spectral.i | 64 +++++++++++++++++ .../executioners/SpectralExecutionerBase.h | 53 +++++++++++++++ include/problems/FFTProblem.h | 19 +++++- include/userobjects/FFTBufferBase.h | 9 +++ include/userobjects/FFTWBufferBase.h | 11 ++- include/userobjects/FourierTransform.h | 1 + include/variables/MooseFFTVariable.h | 11 +-- src/auxkernels/FFTBufferAux.C | 1 + src/executioners/SpectralExecutionerBase.C | 68 +++++++++++++++++++ src/userobjects/FFTBufferBase.C | 5 +- src/userobjects/FFTWBufferBase.C | 22 +++++- 11 files changed, 253 insertions(+), 11 deletions(-) create mode 100644 examples/spectral.i create mode 100644 include/executioners/SpectralExecutionerBase.h create mode 100644 src/executioners/SpectralExecutionerBase.C diff --git a/examples/spectral.i b/examples/spectral.i new file mode 100644 index 00000000..a74290cd --- /dev/null +++ b/examples/spectral.i @@ -0,0 +1,64 @@ +[Mesh] + type = MyTRIMMesh + dim = 2 + xmax = 100 + ymax = 100 + nx = 100 + ny = 100 +[] + +[Problem] + type = FFTProblem +[] + +[AuxVariables] + [./c_aux] + order = CONSTANT + family = MONOMIAL + [./InitialCondition] + type = FunctionIC + function = 'cos(x/100*2*pi*4)*cos(y/100*2*pi*3)' + [../] + [../] +[] + +[Materials] + [./test] + type = ParsedMaterial + args = c + function = c*c + [../] +[] + +[UserObjects] + # Buffers + [./c] + type = RealFFTWBuffer + moose_variable = c_aux + [../] + [./R] + type = RankTwoTensorFFTWBuffer + [../] + + # Solver + # ... +[] + +[AuxKernels] + [./c_aux] + type = FFTBufferAux + variable = c_aux + fft_buffer = c + execute_on = FINAL + [../] +[] + +[Executioner] + type = SpectralExecutionerBase +[] + +[Outputs] + exodus = true + execute_on = 'INITIAL FINAL' + perf_graph = true +[] diff --git a/include/executioners/SpectralExecutionerBase.h b/include/executioners/SpectralExecutionerBase.h new file mode 100644 index 00000000..64fd1008 --- /dev/null +++ b/include/executioners/SpectralExecutionerBase.h @@ -0,0 +1,53 @@ +/**********************************************************************/ +/* DO NOT MODIFY THIS HEADER */ +/* MAGPIE - Mesoscale Atomistic Glue Program for Integrated Execution */ +/* */ +/* Copyright 2017 Battelle Energy Alliance, LLC */ +/* ALL RIGHTS RESERVED */ +/**********************************************************************/ + +#pragma once + +#include "Executioner.h" +#include "FFTWBufferBase.h" +#include "FFTProblem.h" + +// System includes +#include + +// Forward declarations +class InputParameters; + +/** + * FFT Executioner base class. + */ +class SpectralExecutionerBase : public Executioner +{ +public: + static InputParameters validParams(); + + SpectralExecutionerBase(const InputParameters & parameters); + + virtual void init() override; + virtual void execute() override; + virtual bool lastSolveConverged() const override { return true; } + +protected: + template + FFTBufferBase & getFFTBuffer(const std::string & name); + + Real _system_time; + int & _time_step; + Real & _time; + + PerfID _final_timer; + + FFTProblem * _fft_problem; +}; + +template +FFTBufferBase & +SpectralExecutionerBase::getFFTBuffer(const std::string & name) +{ + return _fft_problem->getFFTBuffer("c"); +} diff --git a/include/problems/FFTProblem.h b/include/problems/FFTProblem.h index f0e28674..488aee18 100644 --- a/include/problems/FFTProblem.h +++ b/include/problems/FFTProblem.h @@ -10,6 +10,7 @@ #include "FEProblem.h" #include "AuxiliarySystem.h" +#include "FFTBufferBase.h" /** * Enhanced FEProblem that supports FFT buffers as variables @@ -29,7 +30,8 @@ class FFTProblem : public FEProblem Moose::VarKindType expected_var_type = Moose::VarKindType::VAR_ANY, Moose::VarFieldType expected_var_field_type = Moose::VarFieldType::VAR_FIELD_ANY) override; - // getMaterialData + template + FFTBufferBase & getFFTBuffer(const std::string & name); protected: /// map from variable name to list of variable objects (one per thread) @@ -40,3 +42,18 @@ class FFTProblem : public FEProblem unsigned int _fft_var_number; }; + +template +FFTBufferBase & +FFTProblem::getFFTBuffer(const std::string & name) +{ + std::vector objs; + theWarehouse().query().condition(0).condition(name).queryInto(objs); + if (objs.empty()) + mooseError("Unable to find FFT buffer with name '" + name + "'"); + auto fft_buffer = dynamic_cast *>(objs[0]); + if (!fft_buffer) + mooseError(name, " is not an FFT buffer of the requested type"); + + return *fft_buffer; +} diff --git a/include/userobjects/FFTBufferBase.h b/include/userobjects/FFTBufferBase.h index 2ff31511..556ff9fd 100644 --- a/include/userobjects/FFTBufferBase.h +++ b/include/userobjects/FFTBufferBase.h @@ -13,6 +13,15 @@ template class FFTBufferBase; +#define usingFFTBufferBaseMembers \ + using ElementUserObject::_perf_graph; \ + using FFTBufferBase::_dim; \ + using FFTBufferBase::_grid; \ + using FFTBufferBase::_buffer; \ + using FFTBufferBase::_start; \ + using FFTBufferBase::_stride; \ + using FFTBufferBase::_how_many + /** * Generic FFT interleaved data buffer base class */ diff --git a/include/userobjects/FFTWBufferBase.h b/include/userobjects/FFTWBufferBase.h index 54fc7229..ec1438b1 100644 --- a/include/userobjects/FFTWBufferBase.h +++ b/include/userobjects/FFTWBufferBase.h @@ -10,6 +10,7 @@ #pragma once #include "FFTBufferBase.h" +#include "PerfGraphInterface.h" #include "fftw3.h" @@ -31,9 +32,17 @@ class FFTWBufferBase : public FFTBufferBase void backward() override; protected: - /// FFTW plans + ///@{ FFTW plans fftw_plan _forward_plan; fftw_plan _backward_plan; + ///@} + + ///@{ timers + PerfID _perf_plan; + PerfID _perf_fft; + ///@} + + usingFFTBufferBaseMembers; }; #endif diff --git a/include/userobjects/FourierTransform.h b/include/userobjects/FourierTransform.h index d6795124..e94b806b 100644 --- a/include/userobjects/FourierTransform.h +++ b/include/userobjects/FourierTransform.h @@ -10,6 +10,7 @@ #pragma once #include "ElementUserObject.h" +#include "PerfGraphInterface.h" #include "fftw3.h" #include diff --git a/include/variables/MooseFFTVariable.h b/include/variables/MooseFFTVariable.h index 8107c32b..698eabdc 100644 --- a/include/variables/MooseFFTVariable.h +++ b/include/variables/MooseFFTVariable.h @@ -101,7 +101,7 @@ class MooseFFTVariable : public MooseVariable * @param subdomain The subdomain id in question * @return true if active on subdomain, false otherwise */ - virtual bool activeOnSubdomain(SubdomainID subdomain) const { return true; } + virtual bool activeOnSubdomain(SubdomainID /*subdomain*/) const { return true; } /** * Prepare the initial condition @@ -140,12 +140,15 @@ class MooseFFTVariable : public MooseVariable */ virtual void computeNodalValues() {} - virtual void getDofIndices(const Elem * elem, std::vector & dof_indices) const {} + virtual void getDofIndices(const Elem * /*elem*/, + std::vector & /*dof_indices*/) const + { + } virtual unsigned int numberOfDofsNeighbor() { return 0; } - virtual void insert(NumericVector & residual) {} - virtual void add(NumericVector & residual) {} + virtual void insert(NumericVector & /*residual*/) {} + virtual void add(NumericVector & /*residual*/) {} protected: std::vector _no_dofs; diff --git a/src/auxkernels/FFTBufferAux.C b/src/auxkernels/FFTBufferAux.C index 5207a2d5..31fd7a24 100644 --- a/src/auxkernels/FFTBufferAux.C +++ b/src/auxkernels/FFTBufferAux.C @@ -29,5 +29,6 @@ FFTBufferAux::FFTBufferAux(const InputParameters & parameters) Real FFTBufferAux::computeValue() { + std::cout << 'B'; return _buffer(_current_elem->centroid()); } diff --git a/src/executioners/SpectralExecutionerBase.C b/src/executioners/SpectralExecutionerBase.C new file mode 100644 index 00000000..0f39cbfd --- /dev/null +++ b/src/executioners/SpectralExecutionerBase.C @@ -0,0 +1,68 @@ +/**********************************************************************/ +/* DO NOT MODIFY THIS HEADER */ +/* MAGPIE - Mesoscale Atomistic Glue Program for Integrated Execution */ +/* */ +/* Copyright 2017 Battelle Energy Alliance, LLC */ +/* ALL RIGHTS RESERVED */ +/**********************************************************************/ + +#include "SpectralExecutionerBase.h" +#include "InputParameters.h" + +// testing +registerMooseObject("MagpieApp", SpectralExecutionerBase); + +InputParameters +SpectralExecutionerBase::validParams() +{ + InputParameters params = Executioner::validParams(); + params.addClassDescription("Executioner for FFT simulations."); + params.addParam("time", 0.0, "System time"); + return params; +} + +SpectralExecutionerBase::SpectralExecutionerBase(const InputParameters & parameters) + : Executioner(parameters), + _system_time(getParam("time")), + _time_step(_fe_problem.timeStep()), + _time(_fe_problem.time()), + _final_timer(registerTimedSection("final", 1)), + _fft_problem(dynamic_cast(&_fe_problem)) +{ + + if (!_fft_problem) + mooseError("Use Problem/type=FFTProblem with a spectral executioner"); +} + +void +SpectralExecutionerBase::init() +{ + if (_app.isRecovering()) + { + _console << "\nCannot recover FFT solves!\nExiting...\n" << std::endl; + return; + } + + // checkIntegrity(); + _fe_problem.execute(EXEC_PRE_MULTIAPP_SETUP); + _fe_problem.initialSetup(); +} + +void +SpectralExecutionerBase::execute() +{ + _time_step = 0; + _time = _time_step; + _fe_problem.outputStep(EXEC_INITIAL); + _fe_problem.advanceState(); + + mooseInfo("SpectralExecutionerBase::execute()"); + + auto & c = getFFTBuffer("c"); + c.forward(); + + _time_step = 1; + _fe_problem.execute(EXEC_FINAL); + _time = _time_step; + _fe_problem.outputStep(EXEC_FINAL); +} diff --git a/src/userobjects/FFTBufferBase.C b/src/userobjects/FFTBufferBase.C index b004930b..796846f5 100644 --- a/src/userobjects/FFTBufferBase.C +++ b/src/userobjects/FFTBufferBase.C @@ -111,8 +111,8 @@ FFTBufferBase::FFTBufferBase(const InputParameters & parameters) _buffer.resize(_buffer_size); // compute stride and start pointer - auto istart = start(0); - std::ptrdiff_t istride = reinterpret_cast(start(1)) - reinterpret_cast(istart); + _start = reinterpret_cast(start(0)); + std::ptrdiff_t istride = reinterpret_cast(start(1)) - reinterpret_cast(_start); if (istride % sizeof(Real) != 0) mooseError("Invalid data alignment"); istride /= sizeof(Real); @@ -122,6 +122,7 @@ template <> void FFTBufferBase::execute() { + std::cout << 'A'; // get grid / buffer location Point centroid = _current_elem->centroid(); std::size_t a = 0; diff --git a/src/userobjects/FFTWBufferBase.C b/src/userobjects/FFTWBufferBase.C index 87344bfd..fdeafc52 100644 --- a/src/userobjects/FFTWBufferBase.C +++ b/src/userobjects/FFTWBufferBase.C @@ -16,9 +16,17 @@ template FFTWBufferBase::FFTWBufferBase(const InputParameters & parameters) - : FFTBufferBase(parameters), _forward_plan(nullptr), _backward_plan(nullptr) + : FFTBufferBase(parameters), + _perf_plan(this->registerTimedSection("fftw_plan_r2r", 2)), + _perf_fft(this->registerTimedSection("fftw_execute", 2)) { // create plans + std::vector kind(_dim, FFTW_R2HC); + { + TIME_SECTION(_perf_plan); + _forward_plan = fftw_plan_r2r(_dim, _grid.data(), _start, _start, kind.data(), FFTW_ESTIMATE); + _backward_plan = fftw_plan_r2r(_dim, _grid.data(), _start, _start, kind.data(), FFTW_ESTIMATE); + } } template @@ -33,8 +41,13 @@ template void FFTWBufferBase::forward() { + mooseInfo("FFTWBufferBase::forward()"); + // execute plan - fftw_execute(_forward_plan); + { + TIME_SECTION(_perf_fft); + fftw_execute(_forward_plan); + } } template @@ -42,7 +55,10 @@ void FFTWBufferBase::backward() { // execute plan - fftw_execute(_backward_plan); + { + TIME_SECTION(_perf_fft); + fftw_execute(_backward_plan); + } } // explicit instantiation and registration