From f23b06e7f0ea77e66bad9c4e6057a70634c89bfa Mon Sep 17 00:00:00 2001 From: Joe Wallwork Date: Mon, 9 Dec 2024 15:18:28 +0000 Subject: [PATCH 1/4] Set up ParaGridIO_Xios as a copy of ParaGridIO for now --- core/src/CMakeLists.txt | 8 +- core/src/ParaGridIO.cpp | 4 +- core/src/ParaGridIO_Xios.cpp | 530 ++++++++++++++++++ core/src/StructureFactory.cpp | 7 +- core/src/include/ParaGridIO.hpp | 4 +- core/src/include/ParaGridIO_Xios.hpp | 138 +++++ core/test/CMakeLists.txt | 81 +-- core/test/ParaGrid_test.cpp | 9 +- core/test/XiosReadWrite_test.cpp | 4 +- .../ERA5Atmosphere.cpp | 6 +- .../OceanBoundaryModule/TOPAZOcean.cpp | 8 +- 11 files changed, 747 insertions(+), 52 deletions(-) create mode 100644 core/src/ParaGridIO_Xios.cpp create mode 100644 core/src/include/ParaGridIO_Xios.hpp diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index b3d973f76..0c230d58c 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -1,5 +1,11 @@ # Sources for the main neXtSIM model +if(ENABLE_XIOS) + set(ParaGridIO_Impl "ParaGridIO_Xios") +else() + set(ParaGridIO_Impl "ParaGridIO") +endif() + set(BaseSources "Logged.cpp" "ModelConfig.cpp" @@ -11,7 +17,7 @@ set(BaseSources "CommandLineParser.cpp" "CommonRestartMetadata.cpp" "RectGridIO.cpp" - "ParaGridIO.cpp" + "${ParaGridIO_Impl}.cpp" "FileCallbackCloser.cpp" "Finalizer.cpp" "DevStep.cpp" diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index 63f47cb48..9b6842692 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -1,10 +1,11 @@ /*! * @file ParaGridIO.cpp * - * @date Oct 24, 2022 + * @date 09 Dec 2024 * @author Tim Spain */ +#ifndef USE_XIOS #include "include/ParaGridIO.hpp" #include "include/CommonRestartMetadata.hpp" @@ -513,3 +514,4 @@ void ParaGridIO::closeAllFiles() } } /* namespace Nextsim */ +#endif /* not USE_XIOS */ diff --git a/core/src/ParaGridIO_Xios.cpp b/core/src/ParaGridIO_Xios.cpp new file mode 100644 index 000000000..3aeea3154 --- /dev/null +++ b/core/src/ParaGridIO_Xios.cpp @@ -0,0 +1,530 @@ +/*! + * @file ParaGridIO_Xios.cpp + * + * @date 09 Dec 2024 + * @author Tim Spain + * @author Joe Wallwork + */ + +#ifdef USE_XIOS +#include "include/ParaGridIO_Xios.hpp" + +#include "include/CommonRestartMetadata.hpp" +#include "include/FileCallbackCloser.hpp" +#include "include/Finalizer.hpp" +#include "include/MissingData.hpp" +#include "include/NZLevels.hpp" +#include "include/gridNames.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace Nextsim { + +// TODO: XIOS implementation +ParaGridIO::ParaGridIO(ParametricGrid& grid) + : IParaGridIO(grid) + , openFilesAndIndices(getOpenFilesAndIndices()) + , dimensionKeys({ + // clang-format off + // Accept post-May 2024 (xdim, ydim, zdim) dimension names and pre-May 2024 (x, y, z) + { "yx", ModelArray::Type::H }, + { "ydimxdim", ModelArray::Type::H }, + { "zyx", ModelArray::Type::Z }, + { "zdimydimxdim", ModelArray::Type::Z }, + { "yxdg_comp", ModelArray::Type::DG }, + { "ydimxdimdg_comp", ModelArray::Type::DG }, + { "yxdgstress_comp", ModelArray::Type::DGSTRESS }, + { "ydimxdimdgstress_comp", ModelArray::Type::DGSTRESS }, + { "ycgxcg", ModelArray::Type::CG }, + { "yvertexxvertexncoords", ModelArray::Type::VERTEX }, + // clang-format on + }) + , isDG({ + // clang-format off + { ModelArray::Dimension::X, false }, + { ModelArray::Dimension::Y, false }, + { ModelArray::Dimension::Z, false }, + { ModelArray::Dimension::XCG, true }, + { ModelArray::Dimension::YCG, true }, + { ModelArray::Dimension::DG, true }, + { ModelArray::Dimension::DGSTRESS, true }, + // NCOORDS is a number of components, but not in the same way as the DG components. + { ModelArray::Dimension::NCOORDS, false }, + // clang-format on + }) + , dimCompMap({ + // clang-format off + { ModelArray::componentMap.at(ModelArray::Type::DG), ModelArray::Type::DG }, + { ModelArray::componentMap.at(ModelArray::Type::DGSTRESS), ModelArray::Type::DGSTRESS }, + { ModelArray::componentMap.at(ModelArray::Type::VERTEX), ModelArray::Type::VERTEX }, + // clang-format on + }) +{ + Logged::warning("XIOS integration has not yet been completed"); + static bool doneOnce = doOnce(); +} + +bool ParaGridIO::doOnce() +{ + // TODO: Setup XIOS in this method + + // Register the finalization function here + Finalizer::registerUnique(closeAllFiles); + // Since it should only ever run once, do further one-off initialization: allow distant + // classes to close files via a callback. + FileCallbackCloser::onClose(close); + + return true; +} + +ParaGridIO::~ParaGridIO() = default; + +#ifdef USE_MPI +ModelState ParaGridIO::getModelState(const std::string& filePath, ModelMetadata& metadata) +#else +ModelState ParaGridIO::getModelState(const std::string& filePath) +#endif +{ + // TODO: XIOS implementation + ModelState state; + + try { +#ifdef USE_MPI + netCDF::NcFilePar ncFile(filePath, netCDF::NcFile::read, metadata.mpiComm); +#else + netCDF::NcFile ncFile(filePath, netCDF::NcFile::read); +#endif + netCDF::NcGroup metaGroup(ncFile.getGroup(IStructure::metadataNodeName())); + netCDF::NcGroup dataGroup(ncFile.getGroup(IStructure::dataNodeName())); + + // Dimensions and DG components + std::multimap dimMap = dataGroup.getDims(); + for (auto entry : ModelArray::definedDimensions) { + auto dimType = entry.first; + if (dimCompMap.count(dimType) > 0) + // TODO Assertions that DG in the file equals the compile time DG in the model. See + // #205 + continue; + + ModelArray::DimensionSpec& dimensionSpec = entry.second; + // Find dimensions in the netCDF file by their name in the ModelArray details + netCDF::NcDim dim = dataGroup.getDim(dimensionSpec.name); + // Also check the old name + if (dim.isNull()) { + dim = dataGroup.getDim(dimensionSpec.altName); + } + // If we didn't find a dimension with the dimensions name or altName, throw. + if (dim.isNull()) { + throw std::out_of_range( + std::string("No netCDF dimension found corresponding to the dimension named ") + + dimensionSpec.name + std::string(" or ") + dimensionSpec.altName); + } + if (dimType == ModelArray::Dimension::Z) { + // A special case, as the number of levels in the file might not be + // the number that the selected ice thermodynamics requires. +#ifdef USE_MPI + ModelArray::setDimension(dimType, NZLevels::get(), NZLevels::get(), 0); +#else + ModelArray::setDimension(dimType, NZLevels::get()); +#endif + } else { +#ifdef USE_MPI + auto dimName = dim.getName(); + size_t localLength = 0; + size_t start = 0; + if (dimType == ModelArray::Dimension::X) { + localLength = metadata.localExtentX; + start = metadata.localCornerX; + } else if (dimType == ModelArray::Dimension::Y) { + localLength = metadata.localExtentY; + start = metadata.localCornerY; + } else if (dimType == ModelArray::Dimension::XVERTEX) { + localLength = metadata.localExtentX + 1; + start = metadata.localCornerX; + } else if (dimType == ModelArray::Dimension::YVERTEX) { + localLength = metadata.localExtentY + 1; + start = metadata.localCornerY; + } else { + localLength = dim.getSize(); + start = 0; + } + ModelArray::setDimension(dimType, dim.getSize(), localLength, start); +#else + ModelArray::setDimension(dimType, dim.getSize()); +#endif + } + } + + // Get all vars in the data group, and load them into a new ModelState + + for (auto entry : dataGroup.getVars()) { + const std::string& varName = entry.first; + netCDF::NcVar& var = entry.second; + // Determine the type from the dimensions + std::vector varDims = var.getDims(); + std::string dimKey = ""; + for (netCDF::NcDim& dim : varDims) { + dimKey += dim.getName(); + } + if (!dimensionKeys.count(dimKey)) { + throw std::out_of_range( + std::string("No ModelArray::Type corresponds to the dimensional key ") + + dimKey); + } + ModelArray::Type type = dimensionKeys.at(dimKey); + state.data[varName] = ModelArray(type); + ModelArray& data = state.data.at(varName); + data.resize(); + + std::vector start; + std::vector count; + if (ModelArray::hasDoF(type)) { + auto ncomps = data.nComponents(); + start.push_back(0); + count.push_back(ncomps); + } + for (ModelArray::Dimension dt : ModelArray::typeDimensions.at(type)) { + auto dim = ModelArray::definedDimensions.at(dt); + start.push_back(dim.start); + count.push_back(dim.localLength); + } + // dims are looped in [dg], x, y, [z] order so start and count + // order must be reveresed to match order netcdf expects + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + + var.getVar(start, count, &data[0]); + } + ncFile.close(); + } catch (const netCDF::exceptions::NcException& nce) { + std::string ncWhat(nce.what()); + ncWhat += ": " + filePath; + throw std::runtime_error(ncWhat); + } + return state; +} + +ModelState ParaGridIO::readForcingTimeStatic( + const std::set& forcings, const TimePoint& time, const std::string& filePath) +{ + // TODO: XIOS implementation + + ModelState state; + + try { + netCDF::NcFile ncFile(filePath, netCDF::NcFile::read); + netCDF::NcGroup metaGroup(ncFile.getGroup(IStructure::metadataNodeName())); + netCDF::NcGroup dataGroup(ncFile.getGroup(IStructure::dataNodeName())); + + // Read the time axis + netCDF::NcDim timeDim = dataGroup.getDim(timeName); + // Read the time variable + netCDF::NcVar timeVar = dataGroup.getVar(timeName); + // Calculate the index of the largest time value on the axis below our target + size_t targetTIndex; + // Get the time axis as a vector + std::vector timeVec(timeDim.getSize()); + timeVar.getVar(timeVec.data()); + // Get the index of the largest TimePoint less than the target. + targetTIndex = std::find_if(std::begin(timeVec), std::end(timeVec), [time](double t) { + return (TimePoint() + Duration(t)) > time; + }) - timeVec.begin(); + // Rather than the first that is greater than, get the last that is less + // than or equal to. But don't go out of bounds. + if (targetTIndex > 0) + --targetTIndex; + // ASSUME all forcings are HFields: finite volume fields on the same + // grid as ice thickness + std::vector indexArray = { targetTIndex }; + std::vector extentArray = { 1 }; + + // Loop over the dimensions of H + const std::vector& dimensions + = ModelArray::typeDimensions.at(ModelArray::Type::H); + for (auto riter = dimensions.rbegin(); riter != dimensions.rend(); ++riter) { + indexArray.push_back(0); + extentArray.push_back(ModelArray::definedDimensions.at(*riter).localLength); + } + + for (const std::string& varName : forcings) { + netCDF::NcVar var = dataGroup.getVar(varName); + state.data[varName] = ModelArray(ModelArray::Type::H); + ModelArray& data = state.data.at(varName); + data.resize(); + + var.getVar(indexArray, extentArray, &data[0]); + } + ncFile.close(); + } catch (const netCDF::exceptions::NcException& nce) { + std::string ncWhat(nce.what()); + ncWhat += ": " + filePath; + throw std::runtime_error(ncWhat); + } + return state; +} + +void ParaGridIO::dumpModelState( + const ModelState& state, const ModelMetadata& metadata, const std::string& filePath) +{ + // TODO: XIOS implementation + +#ifdef USE_MPI + netCDF::NcFilePar ncFile(filePath, netCDF::NcFile::replace, metadata.mpiComm); +#else + netCDF::NcFile ncFile(filePath, netCDF::NcFile::replace); +#endif + + CommonRestartMetadata::writeStructureType(ncFile, metadata); + netCDF::NcGroup metaGroup = ncFile.addGroup(IStructure::metadataNodeName()); + netCDF::NcGroup dataGroup = ncFile.addGroup(IStructure::dataNodeName()); + CommonRestartMetadata::writeRestartMetadata(metaGroup, metadata); + + // Dump the dimensions and number of components + std::map ncFromMAMap; + for (auto entry : ModelArray::definedDimensions) { + ModelArray::Dimension dim = entry.first; + size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) + : dimSz = entry.second.globalLength; + ncFromMAMap[dim] = dataGroup.addDim(entry.second.name, dimSz); + // TODO Do I need to add data, even if it is just integers 0...n-1? + } + + // Also create the sets of dimensions to be connected to the data fields + std::map> dimMap; + for (auto entry : ModelArray::typeDimensions) { + ModelArray::Type type = entry.first; + std::vector ncDims; + for (auto iter = entry.second.rbegin(); iter != entry.second.rend(); ++iter) { + ModelArray::Dimension& maDim = *iter; + ncDims.push_back(ncFromMAMap.at(maDim)); + } + dimMap[type] = ncDims; + } + + // Everything that has components needs that dimension, too. This always varies fastest, and so + // is last in the vector of dimensions. + for (auto entry : dimCompMap) { + dimMap.at(entry.second).push_back(ncFromMAMap.at(entry.first)); + } + + std::set restartFields = { hiceName, ciceName, hsnowName, ticeName, sstName, + sssName, maskName, coordsName, xName, yName, longitudeName, latitudeName, gridAzimuthName, + uName, vName, damageName }; // TODO and others + // If the above fields are found in the supplied ModelState, output them + for (auto entry : state.data) { + if (restartFields.count(entry.first)) { + // Get the type, then relevant vector of NetCDF dimensions + ModelArray::Type type = entry.second.getType(); + std::vector start; + std::vector count; + if (ModelArray::hasDoF(type)) { + auto ncomps = entry.second.nComponents(); + start.push_back(0); + count.push_back(ncomps); + } + for (ModelArray::Dimension dt : entry.second.typeDimensions.at(type)) { + auto dim = entry.second.definedDimensions.at(dt); + start.push_back(dim.start); + count.push_back(dim.localLength); + } + // dims are looped in [dg], x, y, [z] order so start and count + // order must be reveresed to match order netcdf expects + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + + std::vector& ncDims = dimMap.at(type); + netCDF::NcVar var(dataGroup.addVar(entry.first, netCDF::ncDouble, ncDims)); + var.putAtt(mdiName, netCDF::ncDouble, MissingData::value()); + var.putVar(start, count, entry.second.getData()); + } + } + + ncFile.close(); +} + +void ParaGridIO::writeDiagnosticTime( + const ModelState& state, const ModelMetadata& meta, const std::string& filePath) +{ + // TODO: XIOS implementation + + bool isNew = openFilesAndIndices.count(filePath) <= 0; + size_t nt = (isNew) ? 0 : ++openFilesAndIndices.at(filePath).second; + if (isNew) { + // Open a new file and emplace it in the map of open files. + // Set the initial time to be zero (assigned above) + // Piecewise construction is necessary to correctly construct the file handle/time index + // pair +#ifdef USE_MPI + openFilesAndIndices.emplace(std::piecewise_construct, std::make_tuple(filePath), + std::forward_as_tuple(std::piecewise_construct, + std::forward_as_tuple(filePath, netCDF::NcFile::replace, meta.mpiComm), + std::forward_as_tuple(nt))); +#else + openFilesAndIndices.emplace(std::piecewise_construct, std::make_tuple(filePath), + std::forward_as_tuple(std::piecewise_construct, + std::forward_as_tuple(filePath, netCDF::NcFile::replace), + std::forward_as_tuple(nt))); +#endif + } + // Get the file handle + NetCDFFileType& ncFile = openFilesAndIndices.at(filePath).first; + + // Get the netCDF groups, creating them if necessary + netCDF::NcGroup metaGroup = (isNew) ? ncFile.addGroup(IStructure::metadataNodeName()) + : ncFile.getGroup(IStructure::metadataNodeName()); + netCDF::NcGroup dataGroup = (isNew) ? ncFile.addGroup(IStructure::dataNodeName()) + : ncFile.getGroup(IStructure::dataNodeName()); + + if (isNew) { + // Write the common structure and time metadata + CommonRestartMetadata::writeStructureType(ncFile, meta); + CommonRestartMetadata::writeRestartMetadata(metaGroup, meta); + } + // Get the unlimited time dimension, creating it if necessary + netCDF::NcDim timeDim = (isNew) ? dataGroup.addDim(timeName) : dataGroup.getDim(timeName); + + // All of the dimensions defined by the data at a particular timestep. + std::map ncFromMAMap; + for (auto entry : ModelArray::definedDimensions) { + ModelArray::Dimension dim = entry.first; + size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) + : dimSz = entry.second.globalLength; + ncFromMAMap[dim] = (isNew) ? dataGroup.addDim(entry.second.name, dimSz) + : dataGroup.getDim(entry.second.name); + } + + // Also create the sets of dimensions to be connected to the data fields + std::map> dimMap; + // Create the index and size arrays + std::map> startMap; + std::map> countMap; + for (auto entry : ModelArray::typeDimensions) { + ModelArray::Type type = entry.first; + std::vector ncDims; + std::vector start; + std::vector count; + + // Everything that has components needs that dimension, too + if (ModelArray::hasDoF(type)) { + if (type == ModelArray::Type::VERTEX && !isNew) + continue; + auto ncomps = ModelArray::nComponents(type); + auto dim = ModelArray::componentMap.at(type); + ncDims.push_back(ncFromMAMap.at(dim)); + start.push_back(0); + count.push_back(ncomps); + } + for (auto dt : entry.second) { + auto dim = ModelArray::definedDimensions.at(dt); + ncDims.push_back(ncFromMAMap.at(dt)); + start.push_back(dim.start); + count.push_back(dim.localLength); + } + + // Deal with VERTEX in each case + // Add the time dimension for all types that are not VERTEX + if (type != ModelArray::Type::VERTEX) { + ncDims.push_back(timeDim); + start.push_back(nt); + count.push_back(1UL); + } else if (!isNew) { + // For VERTEX in an existing file, there is nothing more to be done + continue; + } + + std::reverse(ncDims.begin(), ncDims.end()); + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + + dimMap[type] = ncDims; + startMap[type] = start; + countMap[type] = count; + } + + // Create a special timeless set of dimensions for the landmask + std::vector maskDims; + std::vector maskIndexes; + std::vector maskExtents; + if (isNew) { + for (ModelArray::Dimension& maDim : ModelArray::typeDimensions.at(ModelArray::Type::H)) { + maskDims.push_back(ncFromMAMap.at(maDim)); + } + maskIndexes = { 0, 0 }; + maskExtents = { ModelArray::definedDimensions + .at(ModelArray::typeDimensions.at(ModelArray::Type::H)[0]) + .localLength, + ModelArray::definedDimensions.at(ModelArray::typeDimensions.at(ModelArray::Type::H)[1]) + .localLength }; + } + + // Put the time axis variable + std::vector timeDimVec = { timeDim }; + netCDF::NcVar timeVar((isNew) ? dataGroup.addVar(timeName, netCDF::ncDouble, timeDimVec) + : dataGroup.getVar(timeName)); + double secondsSinceEpoch = (meta.time() - TimePoint()).seconds(); +#ifdef USE_MPI + netCDF::setVariableCollective(timeVar, dataGroup); +#endif + timeVar.putVar({ nt }, { 1 }, &secondsSinceEpoch); + + // Write the data + for (auto entry : state.data) { + ModelArray::Type type = entry.second.getType(); + // Skip timeless fields (mask, coordinates) on existing files + if (!isNew && (entry.first == maskName || type == ModelArray::Type::VERTEX)) + continue; + if (entry.first == maskName) { + // Land mask in a new file (since it was skipped above in existing files) + netCDF::NcVar var(dataGroup.addVar(maskName, netCDF::ncDouble, maskDims)); + // No missing data +#ifdef USE_MPI + netCDF::setVariableCollective(var, dataGroup); +#endif + var.putVar(maskIndexes, maskExtents, entry.second.getData()); + + } else { + std::vector& ncDims = dimMap.at(type); + // Get the variable object, either creating a new one or getting the existing one + netCDF::NcVar var((isNew) ? dataGroup.addVar(entry.first, netCDF::ncDouble, ncDims) + : dataGroup.getVar(entry.first)); + if (isNew) + var.putAtt(mdiName, netCDF::ncDouble, MissingData::value()); +#ifdef USE_MPI + netCDF::setVariableCollective(var, dataGroup); +#endif + var.putVar(startMap.at(type), countMap.at(type), entry.second.getData()); + } + } +} + +// TODO: This method will likely be dropped in the XIOS implementation +void ParaGridIO::close(const std::string& filePath) +{ + if (getOpenFilesAndIndices().count(filePath) > 0) { + getOpenFilesAndIndices().at(filePath).first.close(); + getOpenFilesAndIndices().erase(filePath); + } +} + +// TODO: This method will likely be dropped in the XIOS implementation +void ParaGridIO::closeAllFiles() +{ + std::cout << "ParaGridIO::closeAllFiles: closing " << getOpenFilesAndIndices().size() + << " files" << std::endl; + while (getOpenFilesAndIndices().size() > 0) { + close(getOpenFilesAndIndices().begin()->first); + } +} + +} /* namespace Nextsim */ +#endif /* USE_XIOS */ diff --git a/core/src/StructureFactory.cpp b/core/src/StructureFactory.cpp index fb893e796..e3a5cd269 100644 --- a/core/src/StructureFactory.cpp +++ b/core/src/StructureFactory.cpp @@ -1,7 +1,7 @@ /*! * @file StructureFactory.cpp * - * @date Jan 18, 2022 + * @date 09 Dec 2024 * @author Tim Spain * @author Kacper Kornet */ @@ -13,8 +13,11 @@ #include "include/NextsimModule.hpp" #include "include/RectGridIO.hpp" - +#ifdef USE_XIOS +#include "include/ParaGridIO_Xios.hpp" +#else #include "include/ParaGridIO.hpp" +#endif #include #include diff --git a/core/src/include/ParaGridIO.hpp b/core/src/include/ParaGridIO.hpp index 34e803b24..5db8fe9b0 100644 --- a/core/src/include/ParaGridIO.hpp +++ b/core/src/include/ParaGridIO.hpp @@ -1,10 +1,11 @@ /*! * @file ParaGridIO.hpp * - * @date Oct 24, 2022 + * @date 09 Dec 2024 * @author Tim Spain */ +#ifndef USE_XIOS #ifndef PARAGRIDIO_HPP #define PARAGRIDIO_HPP @@ -125,3 +126,4 @@ class ParaGridIO : public ParametricGrid::IParaGridIO { } /* namespace Nextsim */ #endif /* PARAGRIDIO_HPP */ +#endif /* not USE_XIOS */ diff --git a/core/src/include/ParaGridIO_Xios.hpp b/core/src/include/ParaGridIO_Xios.hpp new file mode 100644 index 000000000..6495fe518 --- /dev/null +++ b/core/src/include/ParaGridIO_Xios.hpp @@ -0,0 +1,138 @@ +/*! + * @file ParaGridIO.hpp + * + * @date 09 Dec 2024 + * @author Tim Spain + */ + +#ifdef USE_XIOS +#ifndef PARAGRIDIO_HPP +#define PARAGRIDIO_HPP + +#include "StructureModule/include/ParametricGrid.hpp" +#include "include/Xios.hpp" + +#ifdef USE_MPI +#include "ParallelNetcdfFile.hpp" +#endif +#include +#include +#include + +namespace Nextsim { + +/*! + * A class to perform input and output for the ParametricGrid class using XIOS. + * Unlike the other GridIO classes, this will hold non-restart files open to + * accumulate data until closed using the close function. + */ +class ParaGridIO : public ParametricGrid::IParaGridIO { +public: +// TODO: This member will likely be dropped in the XIOS implementation +#ifdef USE_MPI + using NetCDFFileType = netCDF::NcFilePar; +#else + using NetCDFFileType = netCDF::NcFile; +#endif + + ParaGridIO(ParametricGrid& grid); + virtual ~ParaGridIO(); + + /*! + * Retrieves the ModelState from a restart file of the parametric_grid type + * using XIOS. + * + * @param filePath The file path containing the file to be read. + */ +#ifdef USE_MPI + ModelState getModelState(const std::string& filePath, ModelMetadata& metadata) override; +#else + ModelState getModelState(const std::string& filePath) override; +#endif + + /*! + * @brief Writes the ModelState to a given file location from the provided + * model data and metadata using XIOS. + * + * @params state The model state and configuration object. + * @params metadata The model metadata (principally the initial file + * creation model time). + * @params filePath The path for the restart file. + */ + void dumpModelState( + const ModelState& state, const ModelMetadata& meta, const std::string& filePath) override; + + /*! + * @brief Reads forcings from a ParameticGrid flavoured file using XIOS. + * + * @param forcings The names of the forcings required. + * @param time The time for which to get the forcings. + * @param filePath Path to the file to read. + */ + ModelState readForcingTime(const std::set& forcings, const TimePoint& time, + const std::string& filePath) override + { + return readForcingTimeStatic(forcings, time, filePath); + } + + /*! + * @brief Writes diagnostic data to a file using XIOS. + * + * @param state The state to write to the file. + * @param time The time of the passed data. + * @param filePath Path of the file to write to. + */ + void writeDiagnosticTime( + const ModelState& state, const ModelMetadata& meta, const std::string& filePath) override; + + // TODO: This method will likely be dropped in the XIOS implementation + /*! + * Closes an open diagnostic file Does nothing when provided with a + * restart file name. + * + * @param filePath The path to the file to be closed. + */ + static void close(const std::string& filePath); + + static ModelState readForcingTimeStatic( + const std::set& forcings, const TimePoint& time, const std::string& filePath); + +private: + typedef std::map> FileAndIndexMap; + + ParaGridIO() = delete; + ParaGridIO(const ParaGridIO& other) = delete; + ParaGridIO& operator=(const ParaGridIO& other) = delete; + + // TODO: These members will likely be dropped in the XIOS implementation + const std::map dimensionKeys; + const std::map isDG; + const std::map dimCompMap; + + // Ensures that static variables are created in the correct order. + // TODO: This method will likely be dropped in the XIOS implementation + static void makeDimCompMap(); + + // Closes all still-open NetCDF files + // TODO: This method will likely be dropped in the XIOS implementation + static void closeAllFiles(); + + // Existing or open files are a property of the computer outside the individual + // class instance, so they are singletons. + // TODO: This member will likely be dropped in the XIOS implementation + FileAndIndexMap& openFilesAndIndices; + // TODO: This method will likely be dropped in the XIOS implementation + inline static FileAndIndexMap& getOpenFilesAndIndices() + { + static FileAndIndexMap fim; + return fim; + } + + //! Performs some one-time initialization for the class. Returns true. + static bool doOnce(); +}; + +} /* namespace Nextsim */ + +#endif /* PARAGRIDIO_HPP */ +#endif /* USE_XIOS */ diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index ed8e02992..e43f5199e 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -38,39 +38,6 @@ if(ENABLE_MPI) DEPENDS partition_metadata_3.nc partition_metadata_2.nc ) - add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp") - target_compile_definitions( - testRectGrid_MPI3 - PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" - ) - target_include_directories( - testRectGrid_MPI3 - PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" - ) - target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest) - - add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp") - target_compile_definitions( - testParaGrid_MPI2 - PRIVATE - USE_MPI - TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" - TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\" - ) - target_include_directories( - testParaGrid_MPI2 - PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" - ) - target_link_libraries(testParaGrid_MPI2 PRIVATE nextsimlib doctest::doctest) - - add_executable(testConfigOutput_MPI2 "ConfigOutput_test.cpp" "MainMPI.cpp") - target_compile_definitions( - testConfigOutput_MPI2 - PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" - ) - target_include_directories(testConfigOutput_MPI2 PRIVATE ${MODEL_INCLUDE_DIR}) - target_link_libraries(testConfigOutput_MPI2 PRIVATE nextsimlib doctest::doctest) - if(ENABLE_XIOS) file( CREATE_LINK @@ -89,12 +56,13 @@ if(ENABLE_MPI) "${xios_EXTERNS}/blitz/" "${xios_EXTERNS}/rapidxml/include" ) + set(MODEL_INCLUDE_DIR "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}") add_executable(testXiosCalendar_MPI2 "XiosCalendar_test.cpp" "MainMPI.cpp") target_compile_definitions(testXiosCalendar_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosCalendar_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosCalendar_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -102,7 +70,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosAxis_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosAxis_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosAxis_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -110,7 +78,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosDomain_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosDomain_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosDomain_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -118,7 +86,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosGrid_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosGrid_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosGrid_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -126,7 +94,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosField_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosField_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosField_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -134,7 +102,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosFile_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosFile_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosFile_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -142,10 +110,43 @@ if(ENABLE_MPI) target_compile_definitions(testXiosReadWrite_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosReadWrite_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosReadWrite_MPI2 PRIVATE nextsimlib doctest::doctest) endif() + + add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp") + target_compile_definitions( + testRectGrid_MPI3 + PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" + ) + target_include_directories( + testRectGrid_MPI3 + PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" + ) + target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest) + + add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp") + target_compile_definitions( + testParaGrid_MPI2 + PRIVATE + USE_MPI + TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" + TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\" + ) + target_include_directories( + testParaGrid_MPI2 + PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" + ) + target_link_libraries(testParaGrid_MPI2 PRIVATE nextsimlib doctest::doctest) + + add_executable(testConfigOutput_MPI2 "ConfigOutput_test.cpp" "MainMPI.cpp") + target_compile_definitions( + testConfigOutput_MPI2 + PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" + ) + target_include_directories(testConfigOutput_MPI2 PRIVATE ${MODEL_INCLUDE_DIR}) + target_link_libraries(testConfigOutput_MPI2 PRIVATE nextsimlib doctest::doctest) else() add_executable(testRectGrid "RectGrid_test.cpp") target_compile_definitions(testRectGrid PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index f4e41e0c1..74f7c6f59 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -1,7 +1,7 @@ /*! * @file ParaGrid_test.cpp * - * @date 24 Sep 2024 + * @date 09 Dec 2024 * @author Tim Spain */ @@ -9,6 +9,7 @@ #include #ifdef USE_MPI #include +#undef INFO #else #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include @@ -17,10 +18,14 @@ #include "include/Configurator.hpp" #include "include/ConfiguredModule.hpp" #include "include/NZLevels.hpp" +#ifdef USE_XIOS +#include "include/ParaGridIO_Xios.hpp" +#else #include "include/ParaGridIO.hpp" -#include "include/ParametricGrid.hpp" +#endif #include "include/IStructure.hpp" #include "include/NextsimModule.hpp" +#include "include/ParametricGrid.hpp" #include "include/gridNames.hpp" #include diff --git a/core/test/XiosReadWrite_test.cpp b/core/test/XiosReadWrite_test.cpp index aaab7da45..51ac403e6 100644 --- a/core/test/XiosReadWrite_test.cpp +++ b/core/test/XiosReadWrite_test.cpp @@ -1,7 +1,7 @@ /*! * @file XiosReadWrite_test.cpp * @author Joe Wallwork - * @date 19 Nov 2024 + * @date 09 Dec 2024 * @brief Tests for XIOS write method * @details * This test is designed to test the read and write methods of the C++ @@ -13,7 +13,7 @@ #include "StructureModule/include/ParametricGrid.hpp" #include "include/NextsimModule.hpp" -#include "include/ParaGridIO.hpp" +#include "include/ParaGridIO_Xios.hpp" #include "include/Xios.hpp" #include diff --git a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp index 75ee026f7..011bb21f7 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp @@ -1,7 +1,7 @@ /*! * @file ERA5Atmosphere.cpp * - * @date 24 Sep 2024 + * @date 09 Dec 2024 * @author Tim Spain */ @@ -9,7 +9,11 @@ #include "include/Finalizer.hpp" #include "include/NextsimModule.hpp" +#ifdef USE_XIOS +#include "include/ParaGridIO_Xios.hpp" +#else #include "include/ParaGridIO.hpp" +#endif namespace Nextsim { diff --git a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp index 70eb86bae..8137a0ae1 100644 --- a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp @@ -1,17 +1,21 @@ /*! * @file TOPAZOcean.cpp * - * @date 24 Sep 2024 + * @date 09 Dec 2024 * @author Tim Spain */ #include "include/TOPAZOcean.hpp" #include "include/Finalizer.hpp" -#include "include/IIceOceanHeatFlux.hpp" #include "include/IFreezingPoint.hpp" +#include "include/IIceOceanHeatFlux.hpp" #include "include/NextsimModule.hpp" +#ifdef USE_XIOS +#include "include/ParaGridIO_Xios.hpp" +#else #include "include/ParaGridIO.hpp" +#endif #include "include/constants.hpp" namespace Nextsim { From ddd19f3e32f4f5e6b60887aec92378284c984044 Mon Sep 17 00:00:00 2001 From: Joe Wallwork Date: Mon, 9 Dec 2024 18:07:52 +0000 Subject: [PATCH 2/4] Add extra workflow job for testing without XIOS --- .github/workflows/test_suite.yml | 40 ++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index 3056b53e5..f7c502653 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -50,9 +50,43 @@ jobs: cd test python3 ThermoIntegration_test.py ./run_test_jan2010_integration_test.sh python3 - cd - - test-ubuntu-mpi: + test-ubuntu-mpi-noxios: + + runs-on: ubuntu-22.04 + container: + image: ghcr.io/nextsimhub/nextsimdg-dev-env:latest + + steps: + - uses: actions/checkout@v2 + + - name: build and compile with MPI but not XIOS + run: | + . /opt/spack-environment/activate.sh + mkdir -p build && cd build + cmake -DENABLE_MPI=ON -DENABLE_XIOS=OFF -DCMAKE_CXX_COMPILER="$(which mpic++)" .. + make -j 4 + + - name: run MPI tests + run: | + . /opt/spack-environment/activate.sh + apt update + apt install -y wget + cd build + (cd core/test && wget "ftp://ftp.nersc.no/nextsim/netCDF/partition_metadata_1.nc") + for component in core physics dynamics + do + cd $component/test + for file in $(find test* -maxdepth 0 -type f) + do + echo $file + nprocs=$(echo $file | sed -r "s/.*MPI([0-9]+)/\1/") + mpirun --allow-run-as-root --oversubscribe -n $nprocs ./$file + done + cd - + done + + test-ubuntu-mpi-xios: runs-on: ubuntu-22.04 container: @@ -86,7 +120,6 @@ jobs: done cd - done - cd - test-mac-serial: @@ -138,4 +171,3 @@ jobs: cd test python ThermoIntegration_test.py ./run_test_jan2010_integration_test.sh python - cd - From 1291ae453cfc581d90c42d677709715070466f1e Mon Sep 17 00:00:00 2001 From: Joe Wallwork Date: Tue, 10 Dec 2024 10:45:37 +0000 Subject: [PATCH 3/4] Drop ParaGridIO_Xios.hpp --- core/src/CMakeLists.txt | 6 +- core/src/ParaGridIO_Xios.cpp | 5 +- core/src/StructureFactory.cpp | 8 +- core/src/include/ParaGridIO.hpp | 4 +- core/src/include/ParaGridIO_Xios.hpp | 138 ------------------ core/test/ParaGrid_test.cpp | 10 +- core/test/XiosReadWrite_test.cpp | 4 +- .../ERA5Atmosphere.cpp | 6 +- .../OceanBoundaryModule/TOPAZOcean.cpp | 6 +- 9 files changed, 16 insertions(+), 171 deletions(-) delete mode 100644 core/src/include/ParaGridIO_Xios.hpp diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 0c230d58c..97ce23c67 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -1,9 +1,9 @@ # Sources for the main neXtSIM model if(ENABLE_XIOS) - set(ParaGridIO_Impl "ParaGridIO_Xios") + set(ParaGridIO_Impl "ParaGridIO_Xios.cpp") else() - set(ParaGridIO_Impl "ParaGridIO") + set(ParaGridIO_Impl "ParaGridIO.cpp") endif() set(BaseSources @@ -17,7 +17,7 @@ set(BaseSources "CommandLineParser.cpp" "CommonRestartMetadata.cpp" "RectGridIO.cpp" - "${ParaGridIO_Impl}.cpp" + "${ParaGridIO_Impl}" "FileCallbackCloser.cpp" "Finalizer.cpp" "DevStep.cpp" diff --git a/core/src/ParaGridIO_Xios.cpp b/core/src/ParaGridIO_Xios.cpp index 3aeea3154..902ed0a45 100644 --- a/core/src/ParaGridIO_Xios.cpp +++ b/core/src/ParaGridIO_Xios.cpp @@ -1,17 +1,18 @@ /*! * @file ParaGridIO_Xios.cpp * - * @date 09 Dec 2024 + * @date 10 Dec 2024 * @author Tim Spain * @author Joe Wallwork */ #ifdef USE_XIOS -#include "include/ParaGridIO_Xios.hpp" +#include "include/ParaGridIO.hpp" #include "include/CommonRestartMetadata.hpp" #include "include/FileCallbackCloser.hpp" #include "include/Finalizer.hpp" +#include "include/Logged.hpp" #include "include/MissingData.hpp" #include "include/NZLevels.hpp" #include "include/gridNames.hpp" diff --git a/core/src/StructureFactory.cpp b/core/src/StructureFactory.cpp index e3a5cd269..f41b95441 100644 --- a/core/src/StructureFactory.cpp +++ b/core/src/StructureFactory.cpp @@ -1,7 +1,7 @@ /*! * @file StructureFactory.cpp * - * @date 09 Dec 2024 + * @date 10 Dec 2024 * @author Tim Spain * @author Kacper Kornet */ @@ -12,12 +12,8 @@ #include "include/IStructure.hpp" #include "include/NextsimModule.hpp" -#include "include/RectGridIO.hpp" -#ifdef USE_XIOS -#include "include/ParaGridIO_Xios.hpp" -#else #include "include/ParaGridIO.hpp" -#endif +#include "include/RectGridIO.hpp" #include #include diff --git a/core/src/include/ParaGridIO.hpp b/core/src/include/ParaGridIO.hpp index 5db8fe9b0..89fc4ec02 100644 --- a/core/src/include/ParaGridIO.hpp +++ b/core/src/include/ParaGridIO.hpp @@ -1,11 +1,10 @@ /*! * @file ParaGridIO.hpp * - * @date 09 Dec 2024 + * @date 10 Dec 2024 * @author Tim Spain */ -#ifndef USE_XIOS #ifndef PARAGRIDIO_HPP #define PARAGRIDIO_HPP @@ -126,4 +125,3 @@ class ParaGridIO : public ParametricGrid::IParaGridIO { } /* namespace Nextsim */ #endif /* PARAGRIDIO_HPP */ -#endif /* not USE_XIOS */ diff --git a/core/src/include/ParaGridIO_Xios.hpp b/core/src/include/ParaGridIO_Xios.hpp deleted file mode 100644 index 6495fe518..000000000 --- a/core/src/include/ParaGridIO_Xios.hpp +++ /dev/null @@ -1,138 +0,0 @@ -/*! - * @file ParaGridIO.hpp - * - * @date 09 Dec 2024 - * @author Tim Spain - */ - -#ifdef USE_XIOS -#ifndef PARAGRIDIO_HPP -#define PARAGRIDIO_HPP - -#include "StructureModule/include/ParametricGrid.hpp" -#include "include/Xios.hpp" - -#ifdef USE_MPI -#include "ParallelNetcdfFile.hpp" -#endif -#include -#include -#include - -namespace Nextsim { - -/*! - * A class to perform input and output for the ParametricGrid class using XIOS. - * Unlike the other GridIO classes, this will hold non-restart files open to - * accumulate data until closed using the close function. - */ -class ParaGridIO : public ParametricGrid::IParaGridIO { -public: -// TODO: This member will likely be dropped in the XIOS implementation -#ifdef USE_MPI - using NetCDFFileType = netCDF::NcFilePar; -#else - using NetCDFFileType = netCDF::NcFile; -#endif - - ParaGridIO(ParametricGrid& grid); - virtual ~ParaGridIO(); - - /*! - * Retrieves the ModelState from a restart file of the parametric_grid type - * using XIOS. - * - * @param filePath The file path containing the file to be read. - */ -#ifdef USE_MPI - ModelState getModelState(const std::string& filePath, ModelMetadata& metadata) override; -#else - ModelState getModelState(const std::string& filePath) override; -#endif - - /*! - * @brief Writes the ModelState to a given file location from the provided - * model data and metadata using XIOS. - * - * @params state The model state and configuration object. - * @params metadata The model metadata (principally the initial file - * creation model time). - * @params filePath The path for the restart file. - */ - void dumpModelState( - const ModelState& state, const ModelMetadata& meta, const std::string& filePath) override; - - /*! - * @brief Reads forcings from a ParameticGrid flavoured file using XIOS. - * - * @param forcings The names of the forcings required. - * @param time The time for which to get the forcings. - * @param filePath Path to the file to read. - */ - ModelState readForcingTime(const std::set& forcings, const TimePoint& time, - const std::string& filePath) override - { - return readForcingTimeStatic(forcings, time, filePath); - } - - /*! - * @brief Writes diagnostic data to a file using XIOS. - * - * @param state The state to write to the file. - * @param time The time of the passed data. - * @param filePath Path of the file to write to. - */ - void writeDiagnosticTime( - const ModelState& state, const ModelMetadata& meta, const std::string& filePath) override; - - // TODO: This method will likely be dropped in the XIOS implementation - /*! - * Closes an open diagnostic file Does nothing when provided with a - * restart file name. - * - * @param filePath The path to the file to be closed. - */ - static void close(const std::string& filePath); - - static ModelState readForcingTimeStatic( - const std::set& forcings, const TimePoint& time, const std::string& filePath); - -private: - typedef std::map> FileAndIndexMap; - - ParaGridIO() = delete; - ParaGridIO(const ParaGridIO& other) = delete; - ParaGridIO& operator=(const ParaGridIO& other) = delete; - - // TODO: These members will likely be dropped in the XIOS implementation - const std::map dimensionKeys; - const std::map isDG; - const std::map dimCompMap; - - // Ensures that static variables are created in the correct order. - // TODO: This method will likely be dropped in the XIOS implementation - static void makeDimCompMap(); - - // Closes all still-open NetCDF files - // TODO: This method will likely be dropped in the XIOS implementation - static void closeAllFiles(); - - // Existing or open files are a property of the computer outside the individual - // class instance, so they are singletons. - // TODO: This member will likely be dropped in the XIOS implementation - FileAndIndexMap& openFilesAndIndices; - // TODO: This method will likely be dropped in the XIOS implementation - inline static FileAndIndexMap& getOpenFilesAndIndices() - { - static FileAndIndexMap fim; - return fim; - } - - //! Performs some one-time initialization for the class. Returns true. - static bool doOnce(); -}; - -} /* namespace Nextsim */ - -#endif /* PARAGRIDIO_HPP */ -#endif /* USE_XIOS */ diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 74f7c6f59..8f47d76f9 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -1,7 +1,7 @@ /*! * @file ParaGrid_test.cpp * - * @date 09 Dec 2024 + * @date 10 Dec 2024 * @author Tim Spain */ @@ -17,14 +17,10 @@ #include "include/Configurator.hpp" #include "include/ConfiguredModule.hpp" -#include "include/NZLevels.hpp" -#ifdef USE_XIOS -#include "include/ParaGridIO_Xios.hpp" -#else -#include "include/ParaGridIO.hpp" -#endif #include "include/IStructure.hpp" +#include "include/NZLevels.hpp" #include "include/NextsimModule.hpp" +#include "include/ParaGridIO.hpp" #include "include/ParametricGrid.hpp" #include "include/gridNames.hpp" diff --git a/core/test/XiosReadWrite_test.cpp b/core/test/XiosReadWrite_test.cpp index 51ac403e6..05be6c985 100644 --- a/core/test/XiosReadWrite_test.cpp +++ b/core/test/XiosReadWrite_test.cpp @@ -1,7 +1,7 @@ /*! * @file XiosReadWrite_test.cpp * @author Joe Wallwork - * @date 09 Dec 2024 + * @date 10 Dec 2024 * @brief Tests for XIOS write method * @details * This test is designed to test the read and write methods of the C++ @@ -13,7 +13,7 @@ #include "StructureModule/include/ParametricGrid.hpp" #include "include/NextsimModule.hpp" -#include "include/ParaGridIO_Xios.hpp" +#include "include/ParaGridIO.hpp" #include "include/Xios.hpp" #include diff --git a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp index 011bb21f7..d6acd250a 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp @@ -1,7 +1,7 @@ /*! * @file ERA5Atmosphere.cpp * - * @date 09 Dec 2024 + * @date 10 Dec 2024 * @author Tim Spain */ @@ -9,11 +9,7 @@ #include "include/Finalizer.hpp" #include "include/NextsimModule.hpp" -#ifdef USE_XIOS -#include "include/ParaGridIO_Xios.hpp" -#else #include "include/ParaGridIO.hpp" -#endif namespace Nextsim { diff --git a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp index 8137a0ae1..bb871da3c 100644 --- a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp @@ -1,7 +1,7 @@ /*! * @file TOPAZOcean.cpp * - * @date 09 Dec 2024 + * @date 10 Dec 2024 * @author Tim Spain */ @@ -11,11 +11,7 @@ #include "include/IFreezingPoint.hpp" #include "include/IIceOceanHeatFlux.hpp" #include "include/NextsimModule.hpp" -#ifdef USE_XIOS -#include "include/ParaGridIO_Xios.hpp" -#else #include "include/ParaGridIO.hpp" -#endif #include "include/constants.hpp" namespace Nextsim { From 306f8e954dc3301929f15ebeff1d508ed6d37dff Mon Sep 17 00:00:00 2001 From: Joe Wallwork Date: Tue, 10 Dec 2024 10:48:44 +0000 Subject: [PATCH 4/4] Revert minor changes --- core/src/StructureFactory.cpp | 5 +- core/src/include/ParaGridIO.hpp | 2 +- core/test/CMakeLists.txt | 81 +++++++++---------- core/test/ParaGrid_test.cpp | 7 +- core/test/XiosReadWrite_test.cpp | 2 +- .../ERA5Atmosphere.cpp | 2 +- .../OceanBoundaryModule/TOPAZOcean.cpp | 4 +- 7 files changed, 51 insertions(+), 52 deletions(-) diff --git a/core/src/StructureFactory.cpp b/core/src/StructureFactory.cpp index f41b95441..fb893e796 100644 --- a/core/src/StructureFactory.cpp +++ b/core/src/StructureFactory.cpp @@ -1,7 +1,7 @@ /*! * @file StructureFactory.cpp * - * @date 10 Dec 2024 + * @date Jan 18, 2022 * @author Tim Spain * @author Kacper Kornet */ @@ -12,9 +12,10 @@ #include "include/IStructure.hpp" #include "include/NextsimModule.hpp" -#include "include/ParaGridIO.hpp" #include "include/RectGridIO.hpp" +#include "include/ParaGridIO.hpp" + #include #include #include diff --git a/core/src/include/ParaGridIO.hpp b/core/src/include/ParaGridIO.hpp index 89fc4ec02..34e803b24 100644 --- a/core/src/include/ParaGridIO.hpp +++ b/core/src/include/ParaGridIO.hpp @@ -1,7 +1,7 @@ /*! * @file ParaGridIO.hpp * - * @date 10 Dec 2024 + * @date Oct 24, 2022 * @author Tim Spain */ diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index e43f5199e..ed8e02992 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -38,6 +38,39 @@ if(ENABLE_MPI) DEPENDS partition_metadata_3.nc partition_metadata_2.nc ) + add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp") + target_compile_definitions( + testRectGrid_MPI3 + PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" + ) + target_include_directories( + testRectGrid_MPI3 + PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" + ) + target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest) + + add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp") + target_compile_definitions( + testParaGrid_MPI2 + PRIVATE + USE_MPI + TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" + TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\" + ) + target_include_directories( + testParaGrid_MPI2 + PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" + ) + target_link_libraries(testParaGrid_MPI2 PRIVATE nextsimlib doctest::doctest) + + add_executable(testConfigOutput_MPI2 "ConfigOutput_test.cpp" "MainMPI.cpp") + target_compile_definitions( + testConfigOutput_MPI2 + PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" + ) + target_include_directories(testConfigOutput_MPI2 PRIVATE ${MODEL_INCLUDE_DIR}) + target_link_libraries(testConfigOutput_MPI2 PRIVATE nextsimlib doctest::doctest) + if(ENABLE_XIOS) file( CREATE_LINK @@ -56,13 +89,12 @@ if(ENABLE_MPI) "${xios_EXTERNS}/blitz/" "${xios_EXTERNS}/rapidxml/include" ) - set(MODEL_INCLUDE_DIR "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}") add_executable(testXiosCalendar_MPI2 "XiosCalendar_test.cpp" "MainMPI.cpp") target_compile_definitions(testXiosCalendar_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosCalendar_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosCalendar_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -70,7 +102,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosAxis_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosAxis_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosAxis_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -78,7 +110,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosDomain_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosDomain_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosDomain_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -86,7 +118,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosGrid_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosGrid_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosGrid_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -94,7 +126,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosField_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosField_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosField_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -102,7 +134,7 @@ if(ENABLE_MPI) target_compile_definitions(testXiosFile_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosFile_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosFile_MPI2 PRIVATE nextsimlib doctest::doctest) @@ -110,43 +142,10 @@ if(ENABLE_MPI) target_compile_definitions(testXiosReadWrite_MPI2 PRIVATE USE_XIOS) target_include_directories( testXiosReadWrite_MPI2 - PRIVATE "${MODEL_INCLUDE_DIR}" "${ModulesRoot}/StructureModule" + PRIVATE "${MODEL_INCLUDE_DIR}" "${XIOS_INCLUDE_LIST}" "${ModulesRoot}/StructureModule" ) target_link_libraries(testXiosReadWrite_MPI2 PRIVATE nextsimlib doctest::doctest) endif() - - add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp") - target_compile_definitions( - testRectGrid_MPI3 - PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" - ) - target_include_directories( - testRectGrid_MPI3 - PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" - ) - target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest) - - add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp") - target_compile_definitions( - testParaGrid_MPI2 - PRIVATE - USE_MPI - TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" - TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\" - ) - target_include_directories( - testParaGrid_MPI2 - PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule" - ) - target_link_libraries(testParaGrid_MPI2 PRIVATE nextsimlib doctest::doctest) - - add_executable(testConfigOutput_MPI2 "ConfigOutput_test.cpp" "MainMPI.cpp") - target_compile_definitions( - testConfigOutput_MPI2 - PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" - ) - target_include_directories(testConfigOutput_MPI2 PRIVATE ${MODEL_INCLUDE_DIR}) - target_link_libraries(testConfigOutput_MPI2 PRIVATE nextsimlib doctest::doctest) else() add_executable(testRectGrid "RectGrid_test.cpp") target_compile_definitions(testRectGrid PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 8f47d76f9..f4e41e0c1 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -1,7 +1,7 @@ /*! * @file ParaGrid_test.cpp * - * @date 10 Dec 2024 + * @date 24 Sep 2024 * @author Tim Spain */ @@ -9,7 +9,6 @@ #include #ifdef USE_MPI #include -#undef INFO #else #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include @@ -17,11 +16,11 @@ #include "include/Configurator.hpp" #include "include/ConfiguredModule.hpp" -#include "include/IStructure.hpp" #include "include/NZLevels.hpp" -#include "include/NextsimModule.hpp" #include "include/ParaGridIO.hpp" #include "include/ParametricGrid.hpp" +#include "include/IStructure.hpp" +#include "include/NextsimModule.hpp" #include "include/gridNames.hpp" #include diff --git a/core/test/XiosReadWrite_test.cpp b/core/test/XiosReadWrite_test.cpp index 05be6c985..aaab7da45 100644 --- a/core/test/XiosReadWrite_test.cpp +++ b/core/test/XiosReadWrite_test.cpp @@ -1,7 +1,7 @@ /*! * @file XiosReadWrite_test.cpp * @author Joe Wallwork - * @date 10 Dec 2024 + * @date 19 Nov 2024 * @brief Tests for XIOS write method * @details * This test is designed to test the read and write methods of the C++ diff --git a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp index d6acd250a..75ee026f7 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp @@ -1,7 +1,7 @@ /*! * @file ERA5Atmosphere.cpp * - * @date 10 Dec 2024 + * @date 24 Sep 2024 * @author Tim Spain */ diff --git a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp index bb871da3c..70eb86bae 100644 --- a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp @@ -1,15 +1,15 @@ /*! * @file TOPAZOcean.cpp * - * @date 10 Dec 2024 + * @date 24 Sep 2024 * @author Tim Spain */ #include "include/TOPAZOcean.hpp" #include "include/Finalizer.hpp" -#include "include/IFreezingPoint.hpp" #include "include/IIceOceanHeatFlux.hpp" +#include "include/IFreezingPoint.hpp" #include "include/NextsimModule.hpp" #include "include/ParaGridIO.hpp" #include "include/constants.hpp"