fix(bigquery): Job Library fixes for InsertJob api (#12746)

googleapis · Sep 28, 2023 · 6ed7976 · 6ed7976
1 parent 7e6cc7b
commit 6ed7976
Show file tree

Hide file tree

Showing 10 changed files with 150 additions and 72 deletions.
diff --git a/google/cloud/bigquery/v2/minimal/internal/job_configuration_query.cc b/google/cloud/bigquery/v2/minimal/internal/job_configuration_query.cc
@@ -54,30 +54,31 @@ std::string JobConfigurationQuery::DebugString(absl::string_view name,
 }
 
 void to_json(nlohmann::json& j, JobConfigurationQuery const& c) {
-  j = nlohmann::json{{"query", c.query},
-                     {"createDisposition", c.create_disposition},
-                     {"writeDisposition", c.write_disposition},
-                     {"priority", c.priority},
-                     {"parameterMode", c.parameter_mode},
-                     {"preserveNulls", c.preserve_nulls},
-                     {"allowLargeResults", c.allow_large_results},
-                     {"useQueryCache", c.use_query_cache},
-                     {"flattenResults", c.flatten_results},
-                     {"useLegacySql", c.use_legacy_sql},
-                     {"createSession", c.create_session},
-                     {"maximumBytesBilled", c.maximum_bytes_billed},
-                     {"queryParameters", c.query_parameters},
-                     {"schemaUpdateOptions", c.schema_update_options},
-                     {"connectionProperties", c.connection_properties},
-                     {"defaultDataset", c.default_dataset},
-                     {"destinationTable", c.destination_table},
-                     {"timePartitioning", c.time_partitioning},
-                     {"rangePartitioning", c.range_partitioning},
-                     {"clustering", c.clustering},
-                     {"destinationEncryptionConfiguration",
-                      c.destination_encryption_configuration},
-                     {"scriptOptions", c.script_options},
-                     {"systemVariables", c.system_variables}};
+  j = nlohmann::json{
+      {"query", c.query},
+      {"createDisposition", c.create_disposition},
+      {"writeDisposition", c.write_disposition},
+      {"priority", c.priority},
+      {"parameterMode", c.parameter_mode},
+      {"preserveNulls", c.preserve_nulls},
+      {"allowLargeResults", c.allow_large_results},
+      {"useQueryCache", c.use_query_cache},
+      {"flattenResults", c.flatten_results},
+      {"useLegacySql", c.use_legacy_sql},
+      {"createSession", c.create_session},
+      {"maximumBytesBilled", std::to_string(c.maximum_bytes_billed)},
+      {"queryParameters", c.query_parameters},
+      {"schemaUpdateOptions", c.schema_update_options},
+      {"connectionProperties", c.connection_properties},
+      {"defaultDataset", c.default_dataset},
+      {"destinationTable", c.destination_table},
+      {"timePartitioning", c.time_partitioning},
+      {"rangePartitioning", c.range_partitioning},
+      {"clustering", c.clustering},
+      {"destinationEncryptionConfiguration",
+       c.destination_encryption_configuration},
+      {"scriptOptions", c.script_options},
+      {"systemVariables", c.system_variables}};
 }
 void from_json(nlohmann::json const& j, JobConfigurationQuery& c) {
   SafeGetTo(c.query, j, "query");
@@ -91,7 +92,7 @@ void from_json(nlohmann::json const& j, JobConfigurationQuery& c) {
   SafeGetTo(c.flatten_results, j, "flattenResults");
   SafeGetTo(c.use_legacy_sql, j, "useLegacySql");
   SafeGetTo(c.create_session, j, "createSession");
-  SafeGetTo(c.maximum_bytes_billed, j, "maximumBytesBilled");
+  c.maximum_bytes_billed = GetNumberFromJson(j, "maximumBytesBilled");
   SafeGetTo(c.query_parameters, j, "queryParameters");
   SafeGetTo(c.schema_update_options, j, "schemaUpdateOptions");
   SafeGetTo(c.connection_properties, j, "connectionProperties");

diff --git a/google/cloud/bigquery/v2/minimal/internal/job_request.cc b/google/cloud/bigquery/v2/minimal/internal/job_request.cc
@@ -227,7 +227,7 @@ StatusOr<rest_internal::RestRequest> BuildRestRequest(
 
   auto const& job = json_payload.get<Job>();
 
-  if (job.configuration.job_type.empty() || job.id != r.job().id) {
+  if (job.configuration.query.query.empty()) {
     return internal::InvalidArgumentError(
         "Invalid InsertJobRequest: Invalid Job object", GCP_ERROR_INFO());
   }

diff --git a/google/cloud/bigquery/v2/minimal/internal/job_request.h b/google/cloud/bigquery/v2/minimal/internal/job_request.h
@@ -227,6 +227,9 @@ class InsertJobRequest {
 
   std::string const& project_id() const { return project_id_; }
   Job const& job() const { return job_; }
+  std::vector<std::string> json_filter_keys() const {
+    return json_filter_keys_;
+  }
 
   InsertJobRequest& set_project_id(std::string project_id) & {
     project_id_ = std::move(project_id);
@@ -244,13 +247,22 @@ class InsertJobRequest {
     return std::move(set_job(std::move(job)));
   }
 
+  InsertJobRequest& set_json_filter_keys(std::vector<std::string> keys) & {
+    json_filter_keys_ = std::move(keys);
+    return *this;
+  }
+  InsertJobRequest&& set_json_filter_keys(std::vector<std::string> keys) && {
+    return std::move(set_json_filter_keys(std::move(keys)));
+  }
+
   std::string DebugString(absl::string_view name,
                           TracingOptions const& options = {},
                           int indent = 0) const;
 
  private:
   std::string project_id_;
   Job job_;
+  std::vector<std::string> json_filter_keys_;
 };
 
 class CancelJobRequest {

diff --git a/google/cloud/bigquery/v2/minimal/internal/job_request_test.cc b/google/cloud/bigquery/v2/minimal/internal/job_request_test.cc
@@ -314,7 +314,7 @@ TEST(InsertJobRequestTest, EmptyProjectId) {
                                HasSubstr("Project Id is empty")));
 }
 
-TEST(InsertJobRequestTest, Invalidjob) {
+TEST(InsertJobRequestTest, EmptyQuery) {
   InsertJobRequest request;
   request.set_project_id("1234");
 

diff --git a/google/cloud/bigquery/v2/minimal/internal/job_response_test.cc b/google/cloud/bigquery/v2/minimal/internal/job_response_test.cc
@@ -282,7 +282,7 @@ TEST(GetJobResponseTest, DebugString) {
       R"( create_disposition: "" write_disposition: "" priority: "")"
       R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
-      R"( create_session: false maximum_bytes_billed: 0)"
+      R"( create_session: false maximum_bytes_billed: -1)"
       R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
       R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning {)"
       R"( type: "" expiration_time { "0" } field: "" } range_partitioning {)"
@@ -335,7 +335,7 @@ TEST(GetJobResponseTest, DebugString) {
       R"( create_disposition: "" write_disposition: "" priority: "")"
       R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
-      R"( create_session: false maximum_bytes_billed: 0)"
+      R"( create_session: false maximum_bytes_billed: -1)"
       R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
       R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
       R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -408,7 +408,7 @@ TEST(GetJobResponseTest, DebugString) {
         flatten_results: false
         use_legacy_sql: false
         create_session: false
-        maximum_bytes_billed: 0
+        maximum_bytes_billed: -1
         default_dataset {
           project_id: ""
           dataset_id: ""
@@ -618,7 +618,7 @@ TEST(ListJobsResponseTest, DebugString) {
       R"( preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false)"
       R"( use_legacy_sql: false create_session: false)"
-      R"( maximum_bytes_billed: 0 default_dataset { project_id: "")"
+      R"( maximum_bytes_billed: -1 default_dataset { project_id: "")"
       R"( dataset_id: "" } destination_table { project_id: "")"
       R"( dataset_id: "" table_id: "" } time_partitioning { type: "")"
       R"( expiration_time { "0" } field: "" } range_partitioning {)"
@@ -672,7 +672,7 @@ TEST(ListJobsResponseTest, DebugString) {
       R"( write_disposition: "" priority: "" parameter_mode: "")"
       R"( preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
-      R"( create_session: false maximum_bytes_billed: 0)"
+      R"( create_session: false maximum_bytes_billed: -1)"
       R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
       R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
       R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -740,7 +740,7 @@ TEST(ListJobsResponseTest, DebugString) {
         flatten_results: false
         use_legacy_sql: false
         create_session: false
-        maximum_bytes_billed: 0
+        maximum_bytes_billed: -1
         default_dataset {
           project_id: ""
           dataset_id: ""
@@ -942,7 +942,7 @@ TEST(InsertJobResponseTest, Success) {
       R"(,"defaultDataset":{"datasetId":"1","projectId":"2"})"
       R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
       R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
-      R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
+      R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
       R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
       R"(,"queryParameters":[{"name":"query-parameter-name")"
       R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"
@@ -1110,7 +1110,7 @@ TEST(InsertJobResponseTest, DebugString) {
       R"( create_disposition: "" write_disposition: "" priority: "")"
       R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
-      R"( create_session: false maximum_bytes_billed: 0)"
+      R"( create_session: false maximum_bytes_billed: -1)"
       R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
       R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning {)"
       R"( type: "" expiration_time { "0" } field: "" } range_partitioning {)"
@@ -1163,7 +1163,7 @@ TEST(InsertJobResponseTest, DebugString) {
       R"( create_disposition: "" write_disposition: "" priority: "")"
       R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
-      R"( create_session: false maximum_bytes_billed: 0)"
+      R"( create_session: false maximum_bytes_billed: -1)"
       R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
       R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
       R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -1237,7 +1237,7 @@ TEST(InsertJobResponseTest, DebugString) {
         flatten_results: false
         use_legacy_sql: false
         create_session: false
-        maximum_bytes_billed: 0
+        maximum_bytes_billed: -1
         default_dataset {
           project_id: ""
           dataset_id: ""
@@ -1424,7 +1424,7 @@ TEST(CancelJobResponseTest, Success) {
       R"(,"defaultDataset":{"datasetId":"1","projectId":"2"})"
       R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
       R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
-      R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
+      R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
       R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
       R"(,"queryParameters":[{"name":"query-parameter-name")"
       R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"
@@ -1611,7 +1611,7 @@ TEST(CancelJobResponseTest, DebugString) {
       R"( create_disposition: "" write_disposition: "" priority: "")"
       R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
-      R"( create_session: false maximum_bytes_billed: 0)"
+      R"( create_session: false maximum_bytes_billed: -1)"
       R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
       R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning {)"
       R"( type: "" expiration_time { "0" } field: "" } range_partitioning {)"
@@ -1666,7 +1666,7 @@ TEST(CancelJobResponseTest, DebugString) {
       R"( create_disposition: "" write_disposition: "" priority: "")"
       R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
       R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
-      R"( create_session: false maximum_bytes_billed: 0)"
+      R"( create_session: false maximum_bytes_billed: -1)"
       R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
       R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
       R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -1741,7 +1741,7 @@ TEST(CancelJobResponseTest, DebugString) {
         flatten_results: false
         use_legacy_sql: false
         create_session: false
-        maximum_bytes_billed: 0
+        maximum_bytes_billed: -1
         default_dataset {
           project_id: ""
           dataset_id: ""

diff --git a/google/cloud/bigquery/v2/minimal/internal/job_rest_stub.cc b/google/cloud/bigquery/v2/minimal/internal/job_rest_stub.cc
@@ -64,11 +64,14 @@ StatusOr<InsertJobResponse> DefaultBigQueryJobRestStub::InsertJob(
   nlohmann::json json_payload;
   to_json(json_payload, request.job());
 
+  auto filtered_json = RemoveJsonKeysAndEmptyFields(json_payload.dump(),
+                                                    request.json_filter_keys());
+
   // 4) Call the rest stub and parse the RestResponse.
   rest_internal::RestContext context;
   return ParseFromRestResponse<InsertJobResponse>(
       rest_stub_->Post(context, std::move(*rest_request),
-                       {absl::MakeConstSpan(json_payload.dump())}));
+                       {absl::MakeConstSpan(filtered_json.dump())}));
 }
 
 StatusOr<CancelJobResponse> DefaultBigQueryJobRestStub::CancelJob(

diff --git a/google/cloud/bigquery/v2/minimal/internal/job_test.cc b/google/cloud/bigquery/v2/minimal/internal/job_test.cc
@@ -1305,7 +1305,7 @@ TEST(JobTest, JobToFromJson) {
       R"(,"defaultDataset":{"datasetId":"1","projectId":"2"})"
       R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
       R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
-      R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
+      R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
       R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
       R"(,"queryParameters":[{"name":"query-parameter-name")"
       R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"
@@ -1430,7 +1430,7 @@ TEST(JobTest, ListFormatJobToFromJson) {
       R"(,"createSession":true,"defaultDataset":{"datasetId":"1","projectId":"2"})"
       R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
       R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
-      R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
+      R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
       R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
       R"(,"queryParameters":[{"name":"query-parameter-name")"
       R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"

diff --git a/google/cloud/bigquery/v2/minimal/internal/json_utils.cc b/google/cloud/bigquery/v2/minimal/internal/json_utils.cc
@@ -85,6 +85,30 @@ void ToJson(std::chrono::system_clock::time_point const& field,
   j[name] = std::to_string(m);
 }
 
+nlohmann::json RemoveJsonKeysAndEmptyFields(
+    std::string const& json_payload, std::vector<std::string> const& keys) {
+  nlohmann::json::parser_callback_t remove_empty_call_back =
+      [keys](int depth, nlohmann::json::parse_event_t event,
+             nlohmann::json& parsed) {
+        (void)depth;
+
+        if (event == nlohmann::json::parse_event_t::key) {
+          auto const discard = std::any_of(
+              keys.begin(), keys.end(),
+              [&parsed](std::string const& key) { return parsed == key; });
+          return !discard;
+        }
+        if (event == nlohmann::json::parse_event_t::object_end) {
+          return parsed != nullptr && !parsed.empty();
+        }
+        if (event == nlohmann::json::parse_event_t::array_end) {
+          return !parsed.empty();
+        }
+        return true;
+      };
+  return nlohmann::json::parse(json_payload, remove_empty_call_back, false);
+}
+
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace bigquery_v2_minimal_internal
 }  // namespace cloud

diff --git a/google/cloud/bigquery/v2/minimal/internal/json_utils.h b/google/cloud/bigquery/v2/minimal/internal/json_utils.h
@@ -44,6 +44,11 @@ void FromJson(std::chrono::hours& field, nlohmann::json const& j,
 void ToJson(std::chrono::hours const& field, nlohmann::json& j,
             char const* name);
 
+// Removes not needed keys and empty arrays and objects from the json
+// payload.
+nlohmann::json RemoveJsonKeysAndEmptyFields(
+    std::string const& json_payload, std::vector<std::string> const& keys = {});
+
 // Suppress recursive clang-tidy warnings
 //
 // NOLINTBEGIN(misc-no-recursion)