feat: list soft deletions (#302)

* adds a new endpoint `/management/v1/warehouse/{warehouse_id}/deleted_tabulars` which lists soft-deleted tabulars in a given warehouse

Cargo.toml

@@ -97,7 +97,7 @@ tower-http = { version = "^0.5", features = [
 tracing = { version = "^0.1", features = ["attributes"] }
 tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] }
 urlencoding = "^2.1"
-utoipa = { version = "4.2.3", features = ["axum_extras"] }
+utoipa = { version = "4.2.3", features = ["axum_extras", "chrono", "url", "uuid"] }
 utoipa-swagger-ui = { version = "7.1.0", features = ["axum"] }
 veil = "0.1.7"
 needs_env_var = "1.0.0"

crates/iceberg-catalog/Cargo.toml

@@ -83,3 +83,4 @@ http-body-util = { workspace = true }
 needs_env_var = { git = "https://github.com/twuebi/needs_env_var.git", rev = "bf14242" }
 tower = { workspace = true }
 tracing-subscriber = { workspace = true }
+serde_urlencoded = "0.7.1"

crates/iceberg-catalog/migrations/20240898175225_tabular_deletion_extended.sql

@@ -0,0 +1,11 @@
+create type deletion_kind as enum ('default', 'purge');
+
+alter table tabular
+ add column deletion_kind deletion_kind;
+
+alter table tabular
+ drop constraint unique_name_per_namespace_id;
+
+
+alter table tabular
+ add constraint unique_name_per_namespace_id unique NULLS not distinct (namespace_id, name, deleted_at);

crates/iceberg-catalog/src/api/iceberg/types.rs

@@ -1,5 +1,6 @@
 //! Helpful types, mostly generated by the axum openapi codegen.
 
+use iceberg_ext::configs::ParseFromStr;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
 #[derive(Debug, Clone, PartialEq, serde::Deserialize)]
@@ -179,13 +180,60 @@ impl<'de> Deserialize<'de> for NextPageToken {
  }
 }
 
+#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub struct DropParams {
+ #[serde(deserialize_with = "deserialize_bool", default)]
+ pub purge_requested: Option<bool>,
+}
+
+fn deserialize_bool<'de, D>(deserializer: D) -> Result<Option<bool>, D::Error>
+where
+ D: Deserializer<'de>,
+{
+ let s: Option<&str> = Option::deserialize(deserializer)?;
+ s.map(<bool>::parse_value)
+ .transpose()
+ .map_err(serde::de::Error::custom)
+}
+
 #[cfg(test)]
 mod tests {
  use super::*;
  use axum::{body::Body, extract::Query, http::Request, routing::get, Router};
  use http_body_util::BodyExt;
  use tower::ServiceExt;
 
+ #[test]
+ fn test_drop_parms() {
+ let query = "purgeRequested=true";
+ let params: DropParams = serde_urlencoded::from_str(query).unwrap();
+ assert_eq!(
+ params,
+ DropParams {
+ purge_requested: Some(true)
+ }
+ );
+
+ let query = "purgeRequested=True";
+ let params: DropParams = serde_urlencoded::from_str(query).unwrap();
+ assert_eq!(
+ params,
+ DropParams {
+ purge_requested: Some(true)
+ }
+ );
+
+ let empty_query = "";
+ let empty_params: DropParams = serde_urlencoded::from_str(empty_query).unwrap();
+ assert_eq!(
+ empty_params,
+ DropParams {
+ purge_requested: None
+ }
+ );
+ }
+
  #[tokio::test]
  async fn test_page_token_de() {
  async fn send_request_get_body(query: &str) -> String {

crates/iceberg-catalog/src/api/iceberg/v1/tables.rs

@@ -1,4 +1,4 @@
-use crate::api::iceberg::types::Prefix;
+use crate::api::iceberg::types::{DropParams, Prefix};
 use crate::api::iceberg::v1::namespace::{NamespaceIdentUrl, NamespaceParameters, PaginationQuery};
 use crate::api::{
  ApiContext, CommitTableRequest, CommitTableResponse, CommitTransactionRequest,
@@ -62,6 +62,7 @@ where
  /// Drop a table from the catalog
  async fn drop_table(
  parameters: TableParameters,
+ drop_params: DropParams,
  state: ApiContext<S>,
  request_metadata: RequestMetadata,
  ) -> Result<()>;
@@ -200,6 +201,7 @@ pub fn router<I: Service<S>, S: crate::api::ThreadSafe>() -> Router<ApiContext<S
  // Drop a table from the catalog
  .delete(
  |Path((prefix, namespace, table)): Path<(Prefix, NamespaceIdentUrl, String)>,
+ Query(drop_params): Query<DropParams>,
  State(api_context): State<ApiContext<S>>,
  Extension(metadata): Extension<RequestMetadata>| async {
  I::drop_table(
@@ -210,6 +212,7 @@ pub fn router<I: Service<S>, S: crate::api::ThreadSafe>() -> Router<ApiContext<S
  name: table,
  },
  },
+ drop_params,
  api_context,
  metadata,
  )

crates/iceberg-catalog/src/api/iceberg/v1/views.rs

@@ -1,4 +1,4 @@
-use crate::api::iceberg::types::Prefix;
+use crate::api::iceberg::types::{DropParams, Prefix};
 use crate::api::iceberg::v1::namespace::{NamespaceIdentUrl, NamespaceParameters, PaginationQuery};
 use crate::api::iceberg::v1::DataAccess;
 use crate::api::{
@@ -60,6 +60,7 @@ where
  /// Remove a view from the catalog
  async fn drop_view(
  parameters: ViewParameters,
+ drop_params: DropParams,
  state: ApiContext<S>,
  request_metadata: RequestMetadata,
  ) -> Result<()>;
@@ -186,6 +187,9 @@ pub fn router<I: Service<S>, S: crate::api::ThreadSafe>() -> Router<ApiContext<S
  name: view,
  },
  },
+ DropParams {
+ purge_requested: None,
+ },
  api_context,
  metadata,
  )