From c89850cde36e5de5acdb077624200a2386ec8fd0 Mon Sep 17 00:00:00 2001 From: Dmitriy Suponitskiy Date: Wed, 16 Oct 2024 21:21:40 -0400 Subject: [PATCH] Corrections to the PRNG changes --- CMakeLists.txt | 50 +++--- configure/config_core.in | 1 - src/core/CMakeLists.txt | 4 +- src/core/include/math/distributiongenerator.h | 147 ++---------------- src/core/include/utils/prng/prng.h | 15 +- src/core/lib/math/distributiongenerator.cpp | 132 +++++++++++++++- src/core/unittest/UnitTestTransform.cpp | 4 +- 7 files changed, 177 insertions(+), 176 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46dd92d26..46973897a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,7 +78,6 @@ option( WITH_NATIVEOPT "Use machine-specific optimizations" option( WITH_COVTEST "Turn on to enable coverage testing" OFF ) option( WITH_NOISE_DEBUG "Use only when running lattice estimator; not for production" OFF ) option( USE_MACPORTS "Use MacPorts installed packages" OFF ) -option( EXTERNAL_PRNG_LIB "Use external PRNG engine" OFF ) # Set required number of bits for native integer in build by setting NATIVE_SIZE to 64 or 128 if( NOT NATIVE_SIZE ) @@ -90,25 +89,24 @@ if( NOT CKKS_M_FACTOR ) endif() ### Print options -message( STATUS "BUILD_UNITTESTS: ${BUILD_UNITTESTS}") -message( STATUS "BUILD_EXAMPLES: ${BUILD_EXAMPLES}") -message( STATUS "BUILD_BENCHMARKS: ${BUILD_BENCHMARKS}") -message( STATUS "BUILD_EXTRAS: ${BUILD_EXTRAS}") -message( STATUS "BUILD_STATIC: ${BUILD_STATIC}") -message( STATUS "BUILD_SHARED: ${BUILD_SHARED}") -message( STATUS "GIT_SUBMOD_AUTO: ${GIT_SUBMOD_AUTO}") -message( STATUS "WITH_BE2: ${WITH_BE2}") -message( STATUS "WITH_BE4: ${WITH_BE4}") -message( STATUS "WITH_NTL: ${WITH_NTL}") -message( STATUS "WITH_TCM: ${WITH_TCM}") -message( STATUS "WITH_OPENMP: ${WITH_OPENMP}") -message( STATUS "NATIVE_SIZE: ${NATIVE_SIZE}") -message( STATUS "CKKS_M_FACTOR: ${CKKS_M_FACTOR}") -message( STATUS "WITH_NATIVEOPT: ${WITH_NATIVEOPT}") -message( STATUS "WITH_COVTEST: ${WITH_COVTEST}") -message( STATUS "WITH_NOISE_DEBUG: ${WITH_NOISE_DEBUG}") -message( STATUS "USE_MACPORTS: ${USE_MACPORTS}") -message( STATUS "EXTERNAL_PRNG_LIB: ${EXTERNAL_PRNG_LIB}") +message( STATUS "BUILD_UNITTESTS: ${BUILD_UNITTESTS}") +message( STATUS "BUILD_EXAMPLES: ${BUILD_EXAMPLES}") +message( STATUS "BUILD_BENCHMARKS: ${BUILD_BENCHMARKS}") +message( STATUS "BUILD_EXTRAS: ${BUILD_EXTRAS}") +message( STATUS "BUILD_STATIC: ${BUILD_STATIC}") +message( STATUS "BUILD_SHARED: ${BUILD_SHARED}") +message( STATUS "GIT_SUBMOD_AUTO: ${GIT_SUBMOD_AUTO}") +message( STATUS "WITH_BE2: ${WITH_BE2}") +message( STATUS "WITH_BE4: ${WITH_BE4}") +message( STATUS "WITH_NTL: ${WITH_NTL}") +message( STATUS "WITH_TCM: ${WITH_TCM}") +message( STATUS "WITH_OPENMP: ${WITH_OPENMP}") +message( STATUS "NATIVE_SIZE: ${NATIVE_SIZE}") +message( STATUS "CKKS_M_FACTOR: ${CKKS_M_FACTOR}") +message( STATUS "WITH_NATIVEOPT: ${WITH_NATIVEOPT}") +message( STATUS "WITH_COVTEST: ${WITH_COVTEST}") +message( STATUS "WITH_NOISE_DEBUG: ${WITH_NOISE_DEBUG}") +message( STATUS "USE_MACPORTS: ${USE_MACPORTS}") #-------------------------------------------------------------------- # Compiler logic @@ -187,6 +185,12 @@ if(WITH_COVTEST) set( COVDIR ${BUILDDIR}coverage/) endif() +if(UNIX AND NOT APPLE) + # OpenFHE may use an external shared object provided by user for PRNG on Linux. In order to ensure that + # OpenFHE can dynamically load shared objects at runtime, add an additional compiler flag: + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ldl") +endif() + if(BUILD_STATIC) set(OpenFHE_STATIC_LIBS OPENFHEcore_static OPENFHEpke_static OPENFHEbinfhe_static) endif() @@ -262,12 +266,6 @@ if(ARCHITECTURE) endif() endif() -if(EXTERNAL_PRNG_LIB) - if(NOT (UNIX AND NOT APPLE)) - message(SEND_ERROR "EXTERNAL_PRNG_LIB is supported for Linux only") - endif() -endif() - # Size checks include(CheckTypeSize) check_type_size("__int128" INT128) diff --git a/configure/config_core.in b/configure/config_core.in index 39918bdb6..d45193ed4 100644 --- a/configure/config_core.in +++ b/configure/config_core.in @@ -9,7 +9,6 @@ #cmakedefine WITH_NOISE_DEBUG #cmakedefine WITH_NTL #cmakedefine WITH_TCM -#cmakedefine EXTERNAL_PRNG_LIB #cmakedefine CKKS_M_FACTOR @CKKS_M_FACTOR@ #cmakedefine HAVE_INT128 @HAVE_INT128@ diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 09206a5a8..5d01340dd 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -46,13 +46,13 @@ install(DIRECTORY include/ add_custom_target( allcore ) if( BUILD_SHARED ) -set (CORELIBS PUBLIC OPENFHEcore ${THIRDPARTYLIBS} ${OpenMP_CXX_FLAGS}) + set (CORELIBS PUBLIC OPENFHEcore ${THIRDPARTYLIBS} ${OpenMP_CXX_FLAGS}) target_link_libraries (OPENFHEcore ${THIRDPARTYLIBS} ${OpenMP_CXX_FLAGS}) add_dependencies( allcore OPENFHEcore) endif() if( BUILD_STATIC ) -set (CORELIBS ${CORELIBS} PUBLIC OPENFHEcore_static ${THIRDPARTYSTATICLIBS} ${OpenMP_CXX_FLAGS}) + set (CORELIBS ${CORELIBS} PUBLIC OPENFHEcore_static ${THIRDPARTYSTATICLIBS} ${OpenMP_CXX_FLAGS}) target_link_libraries (OPENFHEcore_static ${THIRDPARTYSTATICLIBS} ${OpenMP_CXX_FLAGS}) add_dependencies( allcore OPENFHEcore_static) endif() diff --git a/src/core/include/math/distributiongenerator.h b/src/core/include/math/distributiongenerator.h index e71f9f56a..ff87467fa 100644 --- a/src/core/include/math/distributiongenerator.h +++ b/src/core/include/math/distributiongenerator.h @@ -37,19 +37,11 @@ #ifndef __DISTRIBUTIONGENERATOR_H__ #define __DISTRIBUTIONGENERATOR_H__ -#include "config_core.h" #include "utils/prng/prng.h" -#include "utils/prng/blake2engine.h" -#include "utils/exception.h" -#include -#include +#include #include -#include #include -#include -#include -// #include // TODO (dsuponit): delete after testing namespace lbcrypto { // if FIXED_SEED is defined, then PRNG uses a fixed seed number for reproducible results during debug. @@ -65,133 +57,26 @@ namespace lbcrypto { class PseudoRandomNumberGenerator { public: /** - * @brief Returns a reference to the PRNG engine - */ - static PRNG& GetPRNG() { -#pragma omp critical - { - // initialization of PRNGs - if (m_prng == nullptr) { - #ifdef EXTERNAL_PRNG_LIB - // prepare to get the engine from the engine's shared library - // std::cerr << "PRNG library call" << std::endl; - const std::string engineLibName{std::string("lib") + PRNG::PRNGLibName + ".so"}; - void* libraryHandle = dlopen(engineLibName.c_str(), RTLD_LAZY); - if (!libraryHandle) { - std::string errMsg{std::string("Cannot open ") + engineLibName + ": "}; - const char* dlsym_error = dlerror(); - errMsg += dlsym_error; - OPENFHE_THROW(errMsg); - } + * @brief InitPRNGEngine() initializes the PRNG generator + * @param libPath a string with the absolute path to an external PRNG library ("/path/to/libprng.so"). + * If the string is empty, then the default (OpenFHE's built-in PRNG) library will be used. + * @note this function should be called at the beginning of main() if an external library to be used + */ + static void InitPRNGEngine(const std::string& libPath = std::string()); - CreateEngineFuncPtrType createEngine = (CreateEngineFuncPtrType)dlsym(libraryHandle, PRNG::engineFuncName); - #else - // std::cerr << "PRNG default call" << std::endl; - CreateEngineFuncPtrType createEngine = default_prng::createEngineInstance; - #endif - if (!createEngine) { - std::string errMsg{std::string("Cannot load symbol ") + PRNG::engineFuncName}; - #ifdef EXTERNAL_PRNG_LIB - const char* dlsym_error = dlerror(); - errMsg += ": "; - errMsg += dlsym_error; - dlclose(libraryHandle); - #endif - OPENFHE_THROW(errMsg); - } - - std::array seed{}; -#if defined(FIXED_SEED) - // Only used for debugging in the single-threaded mode. - std::cerr << "**FOR DEBUGGING ONLY!!!! Using fixed initializer for PRNG. " - "Use a single thread only, e.g., OMP_NUM_THREADS=1!" - << std::endl; - - seed[0] = 1; -#else - // A 512-bit seed is generated for each thread (this roughly corresponds - // to 256 bits of security). The seed is the sum of a random sample - // generated using std::random_device (typically works correctly in - // Linux, MacOS X, and MinGW starting with GCC 9.2) and a PRNG sample - // seeded from current time stamp, a hash of the current thread, and a - // memory location of a heap variable. The PRNG sample is added in - // case random_device is deterministic (happens on MinGW with GCC - // below 9.2). All future calls to PRNG use the seed generated here. - - // The code below derives randomness from time, thread id, and a memory - // location of a heap variable. This seed is relevant only if the - // implementation of random_device is deterministic (as in older - // versions of GCC in MinGW) - std::array initKey{}; - // high-resolution clock typically has a nanosecond tick period - // Arguably this may give up to 32 bits of entropy as the clock gets - // recycled every 4.3 seconds - initKey[0] = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - // A thread id is often close to being random (on most systems) - initKey[1] = std::hash{}(std::this_thread::get_id()); - // On a 64-bit machine, the thread id is 64 bits long - // skip on 32-bit arm architectures - #if !defined(__arm__) && !defined(__EMSCRIPTEN__) - if (sizeof(size_t) == 8) - initKey[2] = (std::hash{}(std::this_thread::get_id()) >> 32); - #endif - // heap variable; we are going to use the least 32 bits of its memory - // location as the counter. This will increase the entropy of the PRNG sample - void* mem = malloc(1); - uint32_t counter = reinterpret_cast(mem); // NOLINT - free(mem); - - std::uniform_int_distribution distribution(0); - // the code below is wrapped in to {} as we want to get rid of gen immediately after the loop - { - // "PRNG* gen" points at a dynamically allocated (using c++'s new()) memory!!! - std::unique_ptr gen(createEngine(initKey, counter)); - for (auto& s : seed) - s = distribution(*gen); - } - - std::array rdseed{}; - size_t attempts = 3; - bool rdGenPassed = false; - for (size_t i = 0; i < attempts && !rdGenPassed; ++i) { - try { - std::random_device genR; - for (auto& rds : rdseed) { - // we use the fact that there is no overflow for unsigned integers - // (from C++ standard) i.e., arithmetic mod 2^32 is performed. For - // the seed to be random, it is sufficient for one of the two - // samples below to be random. In almost all practical cases, - // distribution(genR) is random. We add distribution(gen) just in - // case there is an implementation issue with random_device (as in - // older MinGW systems). - rds = distribution(genR); - } - rdGenPassed = true; - } - catch (std::exception& e) { - } - } - for (uint32_t i = 0; i < PRNG::MAX_SEED_GENS; ++i) - seed[i] += rdseed[i]; -#endif // FIXED_SEED - m_prng = std::shared_ptr(createEngine(seed, 0)); - // cleanup: close the shared library - #ifdef EXTERNAL_PRNG_LIB - dlclose(libraryHandle); - #endif + /** + * @brief Returns a reference to the PRNG engine + */ + static PRNG& GetPRNG(); - if (!m_prng) - OPENFHE_THROW("Cannot create a PRNG engine"); - } - } // pragma omp critical - return *m_prng; - } private: + using GenPRNGEngineFuncPtr = PRNG* (*)(const std::array&, + PRNG::result_type counter); + // shared pointer to a thread-specific PRNG engine static std::shared_ptr m_prng; - // TODO (dsuponit): add a staic object to handle dlopen/dlsym and initialize it in the source file - using CreateEngineFuncPtrType = PRNG* (*)(const std::array&, - PRNG::result_type counter); + // pointer to the function generating PRNG + static GenPRNGEngineFuncPtr genPRNGEngine; #if !defined(FIXED_SEED) // avoid contention on m_prng: local copies of m_prng are created for each thread diff --git a/src/core/include/utils/prng/prng.h b/src/core/include/utils/prng/prng.h index 78abe8849..2fa1cf944 100644 --- a/src/core/include/utils/prng/prng.h +++ b/src/core/include/utils/prng/prng.h @@ -54,20 +54,11 @@ #include +// ATTENTION (VERY IMPORTANT): +// for any engine class derived from the PRNG class there must be a C function named "createEngineInstance" +// returning a dynamically allocated object of that derived class (see how it is done in blake2engine.h) class PRNG { public: - // ATTENTION (VERY IMPORTANT): - // 1. any PRNG library will be assigned the name specified in the string PRNGLibName, as - // the code opening this library will search for it by that name only. - // 2. for any engine class derived from the PRNG class there must be implemented a C function - // generating a dynamically allocated engine object of that derived class. The name of - // the C function is specified by engineFuncName. - // ========== NEVER(!) CHANGE the values of PRNGLibName and engineFuncName ========== - // the C++ code will construct the full library name out of PRNGLibName ("libPRNGengine.so") - constexpr static const char* PRNGLibName = "PRNGengine"; - constexpr static const char* engineFuncName = "createEngineInstance"; - // ========== NEVER(!) CHANGE the values of PRNGLibName and engineFuncName ========== - enum { MAX_SEED_GENS = 16, // the buffer stores 1024 samples of 32-bit integers diff --git a/src/core/lib/math/distributiongenerator.cpp b/src/core/lib/math/distributiongenerator.cpp index 995a412a7..e95877290 100644 --- a/src/core/lib/math/distributiongenerator.cpp +++ b/src/core/lib/math/distributiongenerator.cpp @@ -35,12 +35,142 @@ */ #include "math/distributiongenerator.h" +#include "utils/prng/blake2engine.h" +#include "utils/exception.h" -#include +#include +#include +#include #include +#include +// #include namespace lbcrypto { std::shared_ptr PseudoRandomNumberGenerator::m_prng = nullptr; +PseudoRandomNumberGenerator::GenPRNGEngineFuncPtr PseudoRandomNumberGenerator::genPRNGEngine = nullptr; + +void PseudoRandomNumberGenerator::InitPRNGEngine(const std::string& libPath) { + if(genPRNGEngine) // if genPRNGEngine has already been initialized + return; + + if(libPath.empty()) { + // use the default OpenFHE PRNG that comes with the library + genPRNGEngine = default_prng::createEngineInstance; + if (!genPRNGEngine) + OPENFHE_THROW("Cannot find symbol: default_prng::createEngineInstance"); + // std::cerr << "InitPRNGEngine: using local PRNG" << std::endl; + } + else { + // do not close libraryHandle, your application will crash if you do + void* libraryHandle = dlopen(libPath.c_str(), RTLD_LAZY); + if (!libraryHandle) { + std::string errMsg{std::string("Cannot open ") + libPath + ": "}; + const char* dlsym_error = dlerror(); + errMsg += dlsym_error; + OPENFHE_THROW(errMsg); + } + genPRNGEngine = (GenPRNGEngineFuncPtr)dlsym(libraryHandle, "createEngineInstance"); + if (!genPRNGEngine) { + std::string errMsg{std::string("Cannot load symbol createEngineInstance() from ") + libPath}; + const char* dlsym_error = dlerror(); + errMsg += ": "; + errMsg += dlsym_error; + dlclose(libraryHandle); + OPENFHE_THROW(errMsg); + } + // std::cerr << "InitPRNGEngine: using external PRNG" << std::endl; + } +} + +PRNG& PseudoRandomNumberGenerator::GetPRNG() { + // initialization of PRNGs + if (m_prng == nullptr) { +#pragma omp critical + { + if (!genPRNGEngine) + InitPRNGEngine(); + + std::array seed{}; + #if defined(FIXED_SEED) + // Only used for debugging in the single-threaded mode. + std::cerr << "**FOR DEBUGGING ONLY!!!! Using fixed initializer for PRNG. " + "Use a single thread only, e.g., OMP_NUM_THREADS=1!" + << std::endl; + + seed[0] = 1; + #else + // A 512-bit seed is generated for each thread (this roughly corresponds + // to 256 bits of security). The seed is the sum of a random sample + // generated using std::random_device (typically works correctly in + // Linux, MacOS X, and MinGW starting with GCC 9.2) and a PRNG sample + // seeded from current time stamp, a hash of the current thread, and a + // memory location of a heap variable. The PRNG sample is added in + // case random_device is deterministic (happens on MinGW with GCC + // below 9.2). All future calls to PRNG use the seed generated here. + + // The code below derives randomness from time, thread id, and a memory + // location of a heap variable. This seed is relevant only if the + // implementation of random_device is deterministic (as in older + // versions of GCC in MinGW) + std::array initKey{}; + // high-resolution clock typically has a nanosecond tick period + // Arguably this may give up to 32 bits of entropy as the clock gets + // recycled every 4.3 seconds + initKey[0] = std::chrono::high_resolution_clock::now().time_since_epoch().count(); + // A thread id is often close to being random (on most systems) + initKey[1] = std::hash{}(std::this_thread::get_id()); + // On a 64-bit machine, the thread id is 64 bits long + // skip on 32-bit arm architectures + #if !defined(__arm__) && !defined(__EMSCRIPTEN__) + if (sizeof(size_t) == 8) + initKey[2] = (std::hash{}(std::this_thread::get_id()) >> 32); + #endif + // heap variable; we are going to use the least 32 bits of its memory + // location as the counter. This will increase the entropy of the PRNG sample + void* mem = malloc(1); + uint32_t counter = reinterpret_cast(mem); // NOLINT + free(mem); + + std::uniform_int_distribution distribution(0); + // the code below is wrapped in to {} as we want to get rid of gen immediately after the loop + { + // "PRNG* gen" points at a dynamically allocated (using c++'s new()) memory!!! + std::unique_ptr gen(genPRNGEngine(initKey, counter)); + for (auto& s : seed) + s = distribution(*gen); + } + + std::array rdseed{}; + size_t attempts = 3; + bool rdGenPassed = false; + for (size_t i = 0; i < attempts && !rdGenPassed; ++i) { + try { + std::random_device genR; + for (auto& rds : rdseed) { + // we use the fact that there is no overflow for unsigned integers + // (from C++ standard) i.e., arithmetic mod 2^32 is performed. For + // the seed to be random, it is sufficient for one of the two + // samples below to be random. In almost all practical cases, + // distribution(genR) is random. We add distribution(gen) just in + // case there is an implementation issue with random_device (as in + // older MinGW systems). + rds = distribution(genR); + } + rdGenPassed = true; + } + catch (std::exception& e) { + } + } + for (uint32_t i = 0; i < PRNG::MAX_SEED_GENS; ++i) + seed[i] += rdseed[i]; + #endif // FIXED_SEED + m_prng = std::shared_ptr(genPRNGEngine(seed, 0)); + if (!m_prng) + OPENFHE_THROW("Cannot create a PRNG engine"); + } // pragma omp critical + } + return *m_prng; +} } // namespace lbcrypto diff --git a/src/core/unittest/UnitTestTransform.cpp b/src/core/unittest/UnitTestTransform.cpp index b652c82dc..2e5803309 100644 --- a/src/core/unittest/UnitTestTransform.cpp +++ b/src/core/unittest/UnitTestTransform.cpp @@ -280,11 +280,9 @@ void CRT_CHECK_big_ring(const std::string& msg) { ChineseRemainderTransformArb().SetCylotomicPolynomial(cycloPoly, modulus); V input(n, modulus); - PRNG gen(1); - std::uniform_int_distribution<> dis(0, 100); // generates a number in [0,100] for (usint i = 0; i < n; i++) { - input.at(i) = typename V::Integer(dis(gen)); + input.at(i) = typename V::Integer(dis(PseudoRandomNumberGenerator::GetPRNG())); } auto output = ChineseRemainderTransformArb().ForwardTransform(input, squareRootOfRoot, bigModulus, bigRoot, m);