feat: create an exportable version of plan loading (#91)

This PR adds an installable target of the plan loading routines (notably
the text format) that is more readily consumed by other languages.

---------

Co-authored-by: Weston Pace <weston.pace@gmail.com>

CMakeLists.txt

@@ -10,6 +10,7 @@ message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
 
 option(SUBSTRAIT_CPP_SANITIZE_DEBUG_BUILD
  "Turns on address and undefined memory sanitization runtime checking."
@@ -55,3 +56,4 @@ if(${SUBSTRAIT_CPP_BUILD_TESTING})
 endif()
 
 add_subdirectory(src/substrait)
+add_subdirectory(export)

export/CMakeLists.txt

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+
+add_subdirectory(planloader)

export/planloader/CMakeLists.txt

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+
+if(NOT BUILD_SUBDIR_NAME EQUAL "release")
+ message(
+ SEND_ERROR,
+ "The planloader library does not work in Debug mode due to its dependencies."
+ )
+endif()
+
+add_library(planloader SHARED planloader.cpp)
+
+add_dependencies(planloader substrait_io)
+target_link_libraries(planloader substrait_io)
+
+install(
+ TARGETS planloader
+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ PRIVATE_HEADER DESTINATION ${CMAKE_INSTALL_INCDIR})
+
+if(${SUBSTRAIT_CPP_BUILD_TESTING})
+ add_subdirectory(tests)
+endif()

export/planloader/planloader.cpp

@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+#include "planloader.h"
+
+#include <limits>
+#include <substrait/common/Io.h>
+
+extern "C" {
+
+// Load a Substrait plan (in any format) from disk.
+// Stores the Substrait plan in planBuffer in serialized form.
+// Returns a SerializedPlan structure containing either the serialized plan or
+// an error message. error_message is nullptr upon success.
+SerializedPlan* load_substrait_plan(const char* filename) {
+ auto newPlan = new SerializedPlan();
+ newPlan->buffer = nullptr;
+ newPlan->size = 0;
+ newPlan->error_message = nullptr;
+
+ auto planOrError = io::substrait::loadPlan(filename);
+ if (!planOrError.ok()) {
+ auto errMsg = planOrError.status().message();
+ newPlan->error_message = new char[errMsg.length()+1];
+ strncpy(newPlan->error_message, errMsg.data(), errMsg.length()+1);
+ return newPlan;
+ }
+ ::substrait::proto::Plan plan = *planOrError;
+ std::string text = plan.SerializeAsString();
+ newPlan->buffer = new unsigned char[text.length()+1];
+ memcpy(newPlan->buffer, text.data(), text.length()+1);
+ newPlan->size = static_cast<int32_t>(
+ text.length() &
+ std::numeric_limits<int32_t>::max());
+ return newPlan;
+}
+
+void free_substrait_plan(SerializedPlan* plan) {
+ delete[] plan->buffer;
+ delete[] plan->error_message;
+ delete plan;
+}
+
+// Write a serialized Substrait plan to disk in the specified format.
+// On error returns a non-empty error message.
+// On success a nullptr is returned.
+const char* save_substrait_plan(
+ const unsigned char* plan_data,
+ int32_t plan_data_length,
+ const char* filename,
+ io::substrait::PlanFileFormat format) {
+ ::substrait::proto::Plan plan;
+ std::string data((const char*) plan_data, plan_data_length);
+ plan.ParseFromString(data);
+ auto result = io::substrait::savePlan(plan, filename, format);
+ if (!result.ok()) {
+ return result.message().data();
+ }
+ return nullptr;
+}
+
+} // extern "C"

export/planloader/planloader.h

@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+#include <substrait/common/Io.h>
+
+extern "C" {
+
+// Since this is actually C code, stick to C style names for exporting.
+// NOLINTBEGIN(readability-identifier-naming)
+
+using SerializedPlan = struct {
+ // If set, contains a serialized ::substrait::proto::Plan object.
+ unsigned char *buffer;
+ // If buffer is set, this is the size of the buffer.
+ int32_t size;
+ // If null the buffer is valid, otherwise this points to a null terminated
+ // error string.
+ char *error_message;
+};
+
+// Load a Substrait plan (in any format) from disk.
+//
+// Accepts filename as a null-terminated C string.
+// Returns a SerializedPlan structure containing either the serialized plan or
+// an error message. This SerializedPlan should be freed using
+// free_substrait_plan.
+SerializedPlan* load_substrait_plan(const char* filename);
+
+// Frees a SerializedPlan that was returned from load_substrait_plan.
+void free_substrait_plan(SerializedPlan* plan);
+
+// Write a serialized Substrait plan to disk in the specified format.
+//
+// plan_data is a Substrait Plan serialized into a byte array with length
+// plan_data_length.
+// Filename is a null-terminated C string.
+// On error returns a non-empty error message.
+// On success an empty string is returned.
+const char* save_substrait_plan(
+ const unsigned char* plan_data,
+ int32_t plan_data_length,
+ const char* filename,
+ io::substrait::PlanFileFormat format);
+
+// NOLINTEND(readability-identifier-naming)
+
+} // extern "C"

export/planloader/tests/CMakeLists.txt

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_path(GET CMAKE_CURRENT_BINARY_DIR PARENT_PATH
+ CMAKE_CURRENT_BINARY_PARENT_DIR)
+
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_PARENT_DIR})
+
+add_test_case(
+ planloader_test
+ SOURCES
+ PlanLoaderTest.cpp
+ EXTRA_LINK_LIBS
+ planloader
+ gmock
+ gtest
+ gtest_main)
+
+set(TEXTPLAN_SOURCE_DIR "${CMAKE_SOURCE_DIR}/src/substrait/textplan")
+
+add_custom_command(
+ TARGET planloader_test
+ POST_BUILD
+ COMMAND ${CMAKE_COMMAND} -E echo "Copying unit test data.."
+ COMMAND ${CMAKE_COMMAND} -E make_directory
+ "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/tests/data"
+ COMMAND
+ ${CMAKE_COMMAND} -E copy
+ "${TEXTPLAN_SOURCE_DIR}/converter/data/q6_first_stage.json"
+ "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/tests/data/q6_first_stage.json")
+
+message(
+ STATUS "test data will be here: ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/tests/data"
+)

export/planloader/tests/PlanLoaderTest.cpp

@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+#include <gmock/gmock-matchers.h>
+#include <gtest/gtest.h>
+#include <functional>
+
+#include "../planloader.h"
+#include "substrait/proto/plan.pb.h"
+
+namespace io::substrait::textplan {
+namespace {
+
+TEST(PlanLoaderTest, LoadAndSave) {
+ auto serializedPlan = load_substrait_plan("data/q6_first_stage.json");
+ ASSERT_EQ(serializedPlan->error_message, nullptr);
+
+ ::substrait::proto::Plan plan;
+ bool parseStatus =
+ plan.ParseFromArray(serializedPlan->buffer, serializedPlan->size);
+ ASSERT_TRUE(parseStatus) << "Failed to parse the plan.";
+
+ const char* saveStatus = save_substrait_plan(
+ serializedPlan->buffer,
+ serializedPlan->size,
+ "outfile.splan",
+ PlanFileFormat::kText);
+ ASSERT_EQ(saveStatus, nullptr);
+
+ free_substrait_plan(serializedPlan);
+}
+
+TEST(PlanLoaderTest, LoadMissingFile) {
+ auto serializedPlan = load_substrait_plan("no_such_file.json");
+ ASSERT_THAT(
+ serializedPlan->error_message,
+ ::testing::StartsWith("Failed to open file no_such_file.json"));
+
+ free_substrait_plan(serializedPlan);
+}
+
+} // namespace
+} // namespace io::substrait::textplan

src/substrait/common/Io.cpp

@@ -55,6 +55,8 @@ absl::StatusOr<::substrait::proto::Plan> loadPlan(
  case PlanFileFormat::kText:
  return textplan::loadFromText(*contentOrError);
  }
+ // There are no other possibilities so this can't happen.
+ return absl::UnimplementedError("Unexpected format encountered.");
 }
 
 absl::Status savePlan(

third_party/antlr4/cmake/ExternalAntlr4Cpp.cmake

@@ -97,6 +97,7 @@ if(ANTLR4_ZIP_REPOSITORY)
  -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
  -DWITH_STATIC_CRT:BOOL=${ANTLR4_WITH_STATIC_CRT}
  -DDISABLE_WARNINGS:BOOL=ON
+ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
  # -DCMAKE_CXX_STANDARD:STRING=17 # if desired, compile the runtime with a different C++ standard
  # -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD} # alternatively, compile the runtime with the same C++ standard as the outer project
  INSTALL_COMMAND ""
@@ -116,6 +117,7 @@ else()
  -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
  -DWITH_STATIC_CRT:BOOL=${ANTLR4_WITH_STATIC_CRT}
  -DDISABLE_WARNINGS:BOOL=ON
+ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
  # -DCMAKE_CXX_STANDARD:STRING=17 # if desired, compile the runtime with a different C++ standard
  # -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD} # alternatively, compile the runtime with the same C++ standard as the outer project
  INSTALL_COMMAND ""