Merge branch 'JoshuaSBrown-synack_w004_42_attempt3' of github.com:ORN…

…L/DataFed into JoshuaSBrown-synack_w004_42_attempt3

CMakeLists.txt

@@ -30,6 +30,27 @@ set(INSTALL_AUTHZ ${BUILD_AUTHZ})
 set(INSTALL_CORE_SERVER ${BUILD_CORE_SERVER})
 set(INSTALL_WEB_SERVER ${BUILD_WEB_SERVER})
 
+function(get_version_from_script INPUT_STRING PATTERN_TO_MATCH OUTPUT_VERSION)
+    string(REGEX MATCH "${PATTERN_TO_MATCH}=(.*)" MATCHED_PART "${INPUT_STRING}")
+    if(MATCHED_PART)
+        string(STRIP "${CMAKE_MATCH_1}" SANITIZED_OUTPUT_VERSION)
+        string(REGEX REPLACE "\"\n.*" "" NEWLINE_REMOVED "${SANITIZED_OUTPUT_VERSION}")
+	string(REPLACE "\"" "" REMOVED_QUOTES "${NEWLINE_REMOVED}")
+	set(${OUTPUT_VERSION} "${REMOVED_QUOTES}" PARENT_SCOPE)
+    endif()
+endfunction()
+
+file(READ "${PROJECT_SOURCE_DIR}/scripts/dependency_versions.sh" DEPENDENCY_VERSIONS)
+
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_PROTOBUF_VERSION" PROTOBUF_VERSION)
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_NLOHMANN_JSON_VERSION" NLOHMANN_JSON_VERSION)
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_LIBZMQ_VERSION" LIBZMQ_VERSION)
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_LIBSODIUM_VERSION" LIBSODIUM_VERSION)
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_JSON_SCHEMA_VALIDATOR_VERSION" JSON_SCHEMA_VALIDATOR_VERSION)
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_OPENSSL" LOCAL_OPENSSL_VERSION)
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_LIBCURL" LOCAL_CURL_VERSION)
+get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_BOOST" LOCAL_BOOST_VERSION)
+
 if ( BUILD_TESTS ) 
   include(CTest)
   ENABLE_TESTING()
@@ -108,29 +129,37 @@ if ( BUILD_REPO_SERVER OR BUILD_CORE_SERVER OR BUILD_AUTHZ OR BUILD_COMMON)
           include_directories( $ENV{BOOST_INC} )
           link_directories( $ENV{BOOST_LIB} )
   else()
-    find_package(Boost REQUIRED COMPONENTS date_time unit_test_framework system filesystem
+	  #set(Boost_USE_STATIC_LIBS   ON)
+    find_package(Boost ${LOCAL_BOOST_VERSION} REQUIRED COMPONENTS date_time unit_test_framework system filesystem
       program_options)
   endif()
 
+  find_package(Threads REQUIRED)
   ## load in pkg-config support
   find_package(PkgConfig)
   ## use pkg-config to get hints for 0mq locations
-  pkg_check_modules(PC_ZeroMQ QUIET zmq)
+  #pkg_check_modules(PC_ZeroMQ QUIET zmq)
+  pkg_check_modules(PkgConfig_ZMQ REQUIRED IMPORTED_TARGET libzmq)
+  #pkg_check_modules(PkgConfig_ZMQ_STATIC REQUIRED IMPORTED_TARGET libzmq-static)
+
+  if(NOT "${LIBZMQ_VERSION}" STREQUAL "${PkgConfig_ZMQ_VERSION}")
+	  message(FATAL_ERROR "Required ZeroMQ version ${LIBZMQ_VERSION}, not satisfied. Found version ${PkgConfig_ZMQ_VERSION}")
+  endif()
 
   ## use the hint from above to find where 'zmq.h' is located
-  find_path(ZeroMQ_INCLUDE_DIR
-          NAMES zmq.h
-          PATHS ${PC_ZeroMQ_INCLUDE_DIRS}
-          )
+  #find_path(ZeroMQ_INCLUDE_DIR
+          #NAMES zmq.h
+          #PATHS ${PC_ZeroMQ_INCLUDE_DIRS}
+          #)
 
   ## use the hint from about to find the location of libzmq
-  find_library(ZeroMQ_LIBRARY
-          NAMES zmq
-          PATHS ${PC_ZeroMQ_LIBRARY_DIRS}
-          )
+  #find_library(ZeroMQ_LIBRARY
+  #        NAMES zmq
+  #        PATHS ${PC_ZeroMQ_LIBRARY_DIRS}
+  #        )
 
-  include_directories( ${ZeroMQ_INCLUDE_DIR} )
-  link_directories( ${PC_ZeroMQ_LIBRARY_DIRS} )
+  #include_directories( ${ZeroMQ_INCLUDE_DIR} )
+  #link_directories( ${PC_ZeroMQ_LIBRARY_DIRS} )
 
   if ( DEFINED ENV{PBUF_INC_DIR} AND DEFINED ENV{PBUF_LIB_DIR})
           message( "PBUF_INC_DIR:" $ENV{PBUF_INC_DIR} )
@@ -140,9 +169,10 @@ if ( BUILD_REPO_SERVER OR BUILD_CORE_SERVER OR BUILD_AUTHZ OR BUILD_COMMON)
     set( Protobuf_LIBRARY $ENV{PBUF_LIB_DIR} )
   endif()
 
-  find_package(Protobuf REQUIRED)
+
+  find_package(Protobuf REQUIRED VERSION "${PROTOBUF_VERSION}" )
   set( PROTOBUF_IMPORT_DIRS ${PROTOBUF_INCLUDE_DIRS})
-  message( TRACE "PROTOBUF_IMPORT_DIRS:" ${PROTOBUF_IMPORT_DIRS} )
+  #message( TRACE "PROTOBUF_IMPORT_DIRS:" ${PROTOBUF_IMPORT_DIRS} )
 
   if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
@@ -167,8 +197,9 @@ endif()
 
 if( BUILD_CORE_SERVER )
   find_package(nlohmann_json CONFIG REQUIRED)
-  find_package(nlohmann_json_schema_validator CONFIG REQUIRED)
-  find_package(OpenSSL REQUIRED)
+  find_package(nlohmann_json_schema_validator CONFIG REQUIRED VERSION "${JSON_SCHEMA_VALIDATOR_VERSION}")
+  find_package(OpenSSL "${LOCAL_OPENSSL_VERSION}" REQUIRED )
+  find_package(CURL "${LOCAL_CURL_VERSION}" REQUIRED )
 endif()
 
 if( BUILD_FOXX OR BUILD_CORE_SERVER )
@@ -227,3 +258,29 @@ if( ENABLE_END_TO_END_TESTS )
     message(WARNING "Cannot run end-to-end tests requires building the python client which has not been enabled")
   endif()
 endif()
+
+# This is not needed but it is used by zmq I think so we want to print what is found locally just in case
+if (PKG_CONFIG_FOUND)
+	pkg_check_modules(sodium_PKG QUIET libsodium)
+endif()
+message("")
+message("Tested Dependency Versions")
+message("             protobuf Version: ${PROTOBUF_VERSION}")
+message("         nlohman json Version: ${NLOHMANN_JSON_VERSION}")
+message("               libzmq Version: ${LIBZMQ_VERSION}")
+message("            libsodium Version: ${LIBSODIUM_VERSION}")
+message("json_schema_validator Version: ${JSON_SCHEMA_VALIDATOR_VERSION}")
+message("                 curl Version: ${LOCAL_CURL_VERSION}")
+message("              openssl Version: ${LOCAL_OPENSSL_VERSION}")
+message("                boost Version: ${LOCAL_BOOST_VERSION}")
+message("")
+message("Found Dependency Versions")
+message("             protobuf Version: ${Protobuf_VERSION}")
+message("         nlohman json Version: ${nlohmann_json_VERSION}")
+message("               libzmq Version: ${PkgConfig_ZMQ_VERSION}")
+message("            libsodium Version: ${sodium_PKG_VERSION}")
+message("json_schema_validator Version: ${nlohmann_json_schema_validator_VERSION}")
+message("                 curl Version: ${CURL_VERSION_STRING}")
+message("              openssl Version: ${OPENSSL_VERSION}")
+message("                boost Version: ${Boost_VERSION}")
+message("")

common/CMakeLists.txt

@@ -19,7 +19,7 @@ if( BUILD_COMMON )
     "source/sockets/*.cpp")
   add_library( common STATIC ${Sources})
   set_target_properties(common PROPERTIES POSITION_INDEPENDENT_CODE ON SOVERSION ${DATAFED_COMMON_LIB_MAJOR} VERSION ${DATAFED_COMMON_LIB_MAJOR}.${DATAFED_COMMON_LIB_MINOR}.${DATAFED_COMMON_LIB_PATCH} )
-  target_link_libraries( common Boost::date_time -lprotobuf datafed-protobuf) 
+  target_link_libraries( common Boost::date_time ${Protobuf_LIBRARIES} datafed-protobuf) 
   target_include_directories( common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include )
   target_include_directories( common PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include )
   target_include_directories( common PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/source )

common/tests/security/tcp_secure/CMakeLists.txt

@@ -1,11 +1,11 @@
 
 include_directories(${CMAKE_SOURCE_DIR}/common/source)
 add_executable(test_tcp_secure_client test_tcp_secure_client.cpp)
-target_link_libraries(test_tcp_secure_client ${Boost_LIBRARIES} common -lzmq -lprotobuf -lpthread)
+target_link_libraries(test_tcp_secure_client ${Boost_LIBRARIES} common PkgConfig::PkgConfig_ZMQ ${Protobuf_LIBRARIES} Threads::Threads)
 target_compile_definitions(test_tcp_secure_client PRIVATE BOOST_TEST_DYN_LINK)
 
 add_executable(test_tcp_secure_server test_tcp_secure_server.cpp)
-target_link_libraries(test_tcp_secure_server ${Boost_LIBRARIES} common -lzmq -lprotobuf -lpthread)
+target_link_libraries(test_tcp_secure_server ${Boost_LIBRARIES} common PkgConfig::PkgConfig_ZMQ ${Protobuf_LIBRARIES} Threads::Threads)
 target_compile_definitions(test_tcp_secure_server PRIVATE BOOST_TEST_DYN_LINK)
 
 find_program(TCPDUMP_CMD NAMES tcpdump)

common/tests/unit/CMakeLists.txt

@@ -17,7 +17,7 @@ foreach(PROG
   include_directories(${PROJECT_SOURCE_DIR}/common/source)
   file(GLOB ${PROG}_SOURCES ${PROG}.cpp)
   add_executable(unit_${PROG} ${${PROG}_SOURCES})
-  target_link_libraries(unit_${PROG} Boost::unit_test_framework Boost::date_time common -lzmq -lprotobuf -lpthread)
+  target_link_libraries(unit_${PROG} ${Boost_LIBRARIES} common PkgConfig::PkgConfig_ZMQ ${Protobuf_LIBRARIES} Threads::Threads)
   target_compile_definitions(unit_${PROG} PRIVATE BOOST_TEST_DYN_LINK)
   add_test(unit_${PROG} unit_${PROG})
 

core/server/CMakeLists.txt

@@ -13,8 +13,8 @@ set_target_properties(datafed-core-lib PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 # Must be public for unit tests to import them
 target_include_directories( datafed-core-lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} )
-target_link_libraries( datafed-core-lib OpenSSL::Crypto common datafed-protobuf -lprotobuf
-  -lpthread -lcurl ${Boost_LIBRARIES} -lzmq -lnlohmann_json_schema_validator )
+target_link_libraries( datafed-core-lib OpenSSL::Crypto common datafed-protobuf ${Protobuf_LIBRARIES}
+	Threads::Threads ${CURL_LIBRARIES} ${Boost_LIBRARIES} PkgConfig::PkgConfig_ZMQ nlohmann_json_schema_validator )
 
 add_executable( datafed-core ${Main})
 target_link_libraries( datafed-core datafed-core-lib )

repository/gridftp/globus5/authz/source/CMakeLists.txt

@@ -9,6 +9,7 @@ add_library( datafed-authz SHARED ${Sources} )
 SET_TARGET_PROPERTIES( datafed-authz PROPERTIES LINKER_LANGUAGE CXX )
 set_target_properties(datafed-authz PROPERTIES POSITION_INDEPENDENT_CODE ON SOVERSION ${DATAFED_AUTHZ_MAJOR} VERSION ${DATAFED_AUTHZ_MAJOR}.${DATAFED_AUTHZ_MINOR}.${DATAFED_AUTHZ_PATCH})
 add_dependencies( datafed-authz common)
-target_link_libraries( datafed-authz common -lprotobuf -lpthread -lzmq ${Boost_LIBRARIES})
+
+target_link_libraries( datafed-authz common ${Protobuf_LIBRARIES} Threads::Threads ${PkgConfig_ZMQ_LIBRARIES} ${Boost_LIBRARIES} )
 
 target_include_directories( datafed-authz PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} )

repository/gridftp/globus5/authz/source/libauthz.c

@@ -392,6 +392,11 @@ globus_result_t gsi_authz_authorize_async(va_list ap) {
   void *callback_arg = va_arg(ap, void *);
   // void *                      authz_system_state  = va_arg(ap, void *);
 
+  char * callout_ids1 = getenv("GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS");
+
+  syslog(LOG_DEBUG,
+    "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n",
+    callout_ids1);
   if (strcmp(action, "lookup") == 0 || strcmp(action, "chdir") == 0) {
     result = GLOBUS_SUCCESS;
     callback(callback_arg, handle, result);

repository/server/CMakeLists.txt

@@ -9,6 +9,6 @@ configure_file(
 
 add_executable( datafed-repo ${Sources} )
 add_dependencies( datafed-repo common )
-target_link_libraries( datafed-repo common -lprotobuf -lpthread -lzmq ${Boost_LIBRARIES} )
+target_link_libraries( datafed-repo common ${Protobuf_LIBRARIES} Threads::Threads ${PkgConfig_ZMQ_LIBRARIES} ${Boost_LIBRARIES} )
 
-target_include_directories( datafed-repo PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} )
+target_include_directories( datafed-repo PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${PkgConfig_ZMQ_INCLUDE_DIRS} ${Protobuf_INCLUDE_DIRS})

scripts/dependency_install_functions.sh

@@ -25,7 +25,7 @@ install_protobuf() {
   git checkout v${DATAFED_PROTOBUF_VERSION}
   git submodule update --init --recursive
   cmake -S cmake/ -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-  cmake --build build -j 4
+  cmake --build build -j 8
   sudo cmake --build build --target install
   cd python
   python3 setup.py build
@@ -58,8 +58,8 @@ install_libzmq() {
   git clone https://github.com/zeromq/libzmq.git
   cd libzmq
   git checkout v${DATAFED_LIBZMQ_VERSION}
-  cmake -S. -B build
-  cmake --build build -j 4
+  cmake -S. -B build -DBUILD_STATIC=ON -DBUILD_SHARED=ON
+  cmake --build build -j 8
   sudo cmake --build build --target install
 }
 
@@ -72,7 +72,7 @@ install_nlohmann_json() {
   cd json
   git checkout v${DATAFED_NLOHMANN_JSON_VERSION}
   cmake -S . -B build
-  cmake --build build -j 4
+  cmake --build build -j 8
   sudo cmake --build build --target install
   cd ../
 }
@@ -86,7 +86,7 @@ install_json_schema_validator() {
   cd json-schema-validator
   git checkout ${DATAFED_JSON_SCHEMA_VALIDATOR_VERSION}
   cmake -S . -B build
-  cmake --build build -j 4
+  cmake --build build -j 8
   sudo cmake --build build --target install
   cd ../
 }

scripts/dependency_versions.sh

@@ -3,8 +3,11 @@
 DATAFED_CMAKE_VERSION="3.17.5"
 DATAFED_GLOBUS_VERSION="6.0.22"
 DATAFED_JSON_SCHEMA_VALIDATOR_VERSION="2.1.0"
+DATAFED_NLOHMANN_JSON_VERSION="3.10.2"
 DATAFED_LIBSODIUM_VERSION="1.0.18"
 DATAFED_LIBZMQ_VERSION="4.3.4"
-DATAFED_NLOHMANN_JSON_VERSION="3.10.2"
 DATAFED_NODE_VERSION="v14.21.3"
 DATAFED_PROTOBUF_VERSION="3.17.3"
+DATAFED_LIBCURL="7.68.0"
+DATAFED_OPENSSL="1.1.1"
+DATAFED_BOOST="1.71.0"

scripts/globus/setup_globus.sh

@@ -30,7 +30,14 @@ COLLECTION_NAME="${DATAFED_GCS_ROOT_NAME} Collection Mapped"
 GUEST_COLLECTION_NAME="${DATAFED_GCS_ROOT_NAME} Collection Guest"
 
 gateway_line=$( globus-connect-server storage-gateway list | grep "$GATEWAY_NAME" )
-
+#    {
+#      "source": "{id}",
+#      "match": "${DATAFED_GLOBUS_APP_ID}@clients.auth.globus.org",
+#      "output": "${DATAFED_GLOBUS_REPO_USER}",
+#      "ignore_case": false,
+#      "literal": true
+#    }
+echo "$DATAFED_GLOBUS_REPO_USER"
 cat << EOF > mapping.json 
 {
   "DATA_TYPE": "expression_identity_mapping#1.0.0",
@@ -41,24 +48,20 @@ cat << EOF > mapping.json
       "output": "${DATAFED_GLOBUS_REPO_USER}",
       "ignore_case": false,
       "literal": false
-    },
-    {
-      "source": "{id}",
-      "match": "${DATAFED_GLOBUS_APP_ID}@clients.auth.globus.org",
-      "output": "${DATAFED_GLOBUS_REPO_USER}",
-      "ignore_case": false,
-      "literal": true
     }
   ]
 }
 EOF
 
+#DOMAINS="--domain ornl.gov --domain clients.auth.globus.org --domain gmail.com"
+DOMAINS="--domain ornl.gov"
+
 if [ -z "$gateway_line" ]
 then
 # Check if it already exists
   globus-connect-server storage-gateway create posix \
     "$GATEWAY_NAME" \
-    --domain ornl.gov --domain clients.auth.globus.org --domain gmail.com \
+    ${DOMAINS}  \
     --identity-mapping file:mapping.json
 
 else
@@ -69,16 +72,16 @@ else
 
   globus-connect-server storage-gateway update posix \
     "$uuid_of_storage_gateway" \
-    --domain ornl.gov --domain clients.auth.globus.org --domain gmail.com \
+    ${DOMAINS} \
     --identity-mapping file:mapping.json
 
 fi
 
 PATH_TO_GUEST_ROOT="${GCS_COLLECTION_ROOT_PATH}"
 
 # Create project/ and /user folders
-mkdir -p "${PATH_TO_GUEST_ROOT}/${DATAFED_REPO_ID_AND_DIR}/user"
-mkdir -p "${PATH_TO_GUEST_ROOT}/${DATAFED_REPO_ID_AND_DIR}/project"
+#mkdir -p "${PATH_TO_GUEST_ROOT}/${DATAFED_REPO_ID_AND_DIR}/user"
+#mkdir -p "${PATH_TO_GUEST_ROOT}/${DATAFED_REPO_ID_AND_DIR}/project"
 
 collection_line=$( globus-connect-server collection list | grep "$COLLECTION_NAME" )
 
@@ -95,6 +98,7 @@ then
     "$COLLECTION_NAME" \
     --allow-guest-collections \
     --enable-anonymous-writes \
+    --default-directory "${GCS_COLLECTION_ROOT_PATH}" \
     --disable-https
 else
 
@@ -104,6 +108,7 @@ else
     "$uuid_of_collection" \
     --allow-guest-collections \
     --enable-anonymous-writes \
+    --default-directory "${GCS_COLLECTION_ROOT_PATH}" \
     --disable-https
 
 fi

scripts/install_core_dependencies.sh

@@ -18,26 +18,26 @@ rapidjson-dev libkrb5-dev git python3-pkg-resources python3-pip libssl-dev
 sudo apt-get install -y libzmq3-dev 
 
 cd ~
-install_cmake
-#cd ~
-#
-## Install cmake 3.17
-#
-#python3 -m pip install --upgrade pip
-#python3 -m pip install setuptools
-#
-#install_nlohmann_json
-#cd ~
-#
-#install_json_schema_validator
-#cd ~
-#
-#install_protobuf
-#cd ~
-#
-#install_libsodium
-#cd ~
-#
-#install_libzmq
-#cd ~
+#install_cmake
+cd ~
+
+# Install cmake 3.17
+
+python3 -m pip install --upgrade pip
+python3 -m pip install setuptools
+
+install_nlohmann_json
+cd ~
+
+install_json_schema_validator
+cd ~
+
+install_protobuf
+cd ~
+
+install_libsodium
+cd ~
+
+install_libzmq
+cd ~