feat: Add ConfigOptions to ScalarFunctionArgs

datafusion-examples/examples/composed_extension_codec.rs

@@ -71,8 +71,10 @@ async fn main() {
 
     // deserialize proto back to execution plan
     let runtime = ctx.runtime_env();
+    let state = ctx.state();
+    let config_options = state.config_options();
     let result_exec_plan: Arc<dyn ExecutionPlan> = proto
-        .try_into_physical_plan(&ctx, runtime.deref(), &composed_codec)
+        .try_into_physical_plan(&ctx, config_options, runtime.deref(), &composed_codec)
         .expect("from proto");
 
     // assert that the original and deserialized execution plans are equal

datafusion-examples/examples/expr_api.rs

@@ -28,6 +28,7 @@ use datafusion::functions_aggregate::first_last::first_value_udaf;
 use datafusion::optimizer::simplify_expressions::ExprSimplifier;
 use datafusion::physical_expr::{analyze, AnalysisContext, ExprBoundaries};
 use datafusion::prelude::*;
+use datafusion_common::config::ConfigOptions;
 use datafusion_common::tree_node::{Transformed, TreeNode};
 use datafusion_common::{ScalarValue, ToDFSchema};
 use datafusion_expr::execution_props::ExecutionProps;
@@ -169,7 +170,8 @@ fn simplify_demo() -> Result<()> {
     // expressions, such as the current time (to evaluate `now()`
     // correctly)
     let props = ExecutionProps::new();
-    let context = SimplifyContext::new(&props).with_schema(schema);
+    let config_options = ConfigOptions::default();
+    let context = SimplifyContext::new(&props, &config_options).with_schema(schema);
     let simplifier = ExprSimplifier::new(context);
 
     // And then call the simplify_expr function:
@@ -184,7 +186,8 @@ fn simplify_demo() -> Result<()> {
 
     // here are some other examples of what DataFusion is capable of
     let schema = Schema::new(vec![make_field("i", DataType::Int64)]).to_dfschema_ref()?;
-    let context = SimplifyContext::new(&props).with_schema(schema.clone());
+    let context =
+        SimplifyContext::new(&props, &config_options).with_schema(schema.clone());
     let simplifier = ExprSimplifier::new(context);
 
     // basic arithmetic simplification
@@ -356,8 +359,13 @@ fn type_coercion_demo() -> Result<()> {
 
     // Evaluation with an expression that has not been type coerced cannot succeed.
     let props = ExecutionProps::default();
-    let physical_expr =
-        datafusion_physical_expr::create_physical_expr(&expr, &df_schema, &props)?;
+    let config_options = ConfigOptions::default();
+    let physical_expr = datafusion_physical_expr::create_physical_expr(
+        &expr,
+        &df_schema,
+        &props,
+        &config_options,
+    )?;
     let e = physical_expr.evaluate(&batch).unwrap_err();
     assert!(e
         .find_root()
@@ -370,13 +378,15 @@ fn type_coercion_demo() -> Result<()> {
     assert!(physical_expr.evaluate(&batch).is_ok());
 
     // 2. Type coercion with `ExprSimplifier::coerce`.
-    let context = SimplifyContext::new(&props).with_schema(Arc::new(df_schema.clone()));
+    let context = SimplifyContext::new(&props, &config_options)
+        .with_schema(Arc::new(df_schema.clone()));
     let simplifier = ExprSimplifier::new(context);
     let coerced_expr = simplifier.coerce(expr.clone(), &df_schema)?;
     let physical_expr = datafusion_physical_expr::create_physical_expr(
         &coerced_expr,
         &df_schema,
         &props,
+        &config_options,
     )?;
     assert!(physical_expr.evaluate(&batch).is_ok());
 
@@ -389,6 +399,7 @@ fn type_coercion_demo() -> Result<()> {
         &coerced_expr,
         &df_schema,
         &props,
+        &config_options,
     )?;
     assert!(physical_expr.evaluate(&batch).is_ok());
 
@@ -417,6 +428,7 @@ fn type_coercion_demo() -> Result<()> {
         &coerced_expr,
         &df_schema,
         &props,
+        &config_options,
     )?;
     assert!(physical_expr.evaluate(&batch).is_ok());
 

datafusion-examples/examples/pruning.rs

@@ -22,6 +22,7 @@ use datafusion::execution::context::ExecutionProps;
 use datafusion::physical_expr::create_physical_expr;
 use datafusion::physical_optimizer::pruning::{PruningPredicate, PruningStatistics};
 use datafusion::prelude::*;
+use datafusion_common::config::ConfigOptions;
 use std::collections::HashSet;
 use std::sync::Arc;
 
@@ -187,7 +188,9 @@ impl PruningStatistics for MyCatalog {
 fn create_pruning_predicate(expr: Expr, schema: &SchemaRef) -> PruningPredicate {
     let df_schema = DFSchema::try_from(schema.as_ref().clone()).unwrap();
     let props = ExecutionProps::new();
-    let physical_expr = create_physical_expr(&expr, &df_schema, &props).unwrap();
+    let config_options = ConfigOptions::default();
+    let physical_expr =
+        create_physical_expr(&expr, &df_schema, &props, &config_options).unwrap();
     PruningPredicate::try_new(physical_expr, schema.clone()).unwrap()
 }
 

datafusion-examples/examples/simple_udtf.rs

@@ -28,6 +28,7 @@ use datafusion::physical_plan::memory::MemoryExec;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
 use datafusion_catalog::TableFunctionImpl;
+use datafusion_common::config::ConfigOptions;
 use datafusion_common::{plan_err, ScalarValue};
 use datafusion_expr::simplify::SimplifyContext;
 use datafusion_expr::{Expr, TableType};
@@ -48,7 +49,9 @@ async fn main() -> Result<()> {
     let ctx = SessionContext::new();
 
     // register the table function that will be called in SQL statements by `read_csv`
-    ctx.register_udtf("read_csv", Arc::new(LocalCsvTableFunc {}));
+    let state = ctx.state();
+    let config_options = Arc::new(state.config_options().clone());
+    ctx.register_udtf("read_csv", Arc::new(LocalCsvTableFunc { config_options }));
 
     let testdata = datafusion::test_util::arrow_test_data();
     let csv_file = format!("{testdata}/csv/aggregate_test_100.csv");
@@ -129,7 +132,9 @@ impl TableProvider for LocalCsvTable {
 }
 
 #[derive(Debug)]
-struct LocalCsvTableFunc {}
+struct LocalCsvTableFunc {
+    config_options: Arc<ConfigOptions>,
+}
 
 impl TableFunctionImpl for LocalCsvTableFunc {
     fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
@@ -142,7 +147,8 @@ impl TableFunctionImpl for LocalCsvTableFunc {
             .map(|expr| {
                 // try to simplify the expression, so 1+2 becomes 3, for example
                 let execution_props = ExecutionProps::new();
-                let info = SimplifyContext::new(&execution_props);
+                let config_options = Arc::unwrap_or_clone(self.config_options.clone());
+                let info = SimplifyContext::new(&execution_props, &config_options);
                 let expr = ExprSimplifier::new(info).simplify(expr.clone())?;
 
                 if let Expr::Literal(ScalarValue::Int64(Some(limit))) = expr {

datafusion/common/src/config.rs

@@ -679,7 +679,7 @@ config_namespace! {
 }
 
 /// A key value pair, with a corresponding description
-#[derive(Debug)]
+#[derive(Debug, Hash, PartialEq, Eq)]
 pub struct ConfigEntry {
     /// A unique string to identify this config value
     pub key: String,

datafusion/core/src/datasource/listing/helpers.rs

@@ -241,6 +241,7 @@ async fn prune_partitions(
     partitions: Vec<Partition>,
     filters: &[Expr],
     partition_cols: &[(String, DataType)],
+    ctx: &SessionState,
 ) -> Result<Vec<Partition>> {
     if filters.is_empty() {
         return Ok(partitions);
@@ -286,13 +287,12 @@ async fn prune_partitions(
     )?;
 
     let batch = RecordBatch::try_new(schema, arrays)?;
-
-    // TODO: Plumb this down
     let props = ExecutionProps::new();
+    let config_options = ctx.config_options();
 
     // Applies `filter` to `batch` returning `None` on error
     let do_filter = |filter| -> Result<ArrayRef> {
-        let expr = create_physical_expr(filter, &df_schema, &props)?;
+        let expr = create_physical_expr(filter, &df_schema, &props, config_options)?;
         expr.evaluate(&batch)?.into_array(partitions.len())
     };
 
@@ -436,7 +436,7 @@ pub async fn pruned_partition_list<'a>(
     debug!("Listed {} partitions", partitions.len());
 
     let pruned =
-        prune_partitions(table_path, partitions, filters, partition_cols).await?;
+        prune_partitions(table_path, partitions, filters, partition_cols, ctx).await?;
 
     debug!("Pruning yielded {} partitions", pruned.len());
 

datafusion/core/src/datasource/listing/table.rs

@@ -894,6 +894,7 @@ impl TableProvider for ListingTable {
                     &expr,
                     &table_df_schema,
                     state.execution_props(),
+                    session_state.config_options(),
                 )?;
                 Some(filters)
             }

datafusion/core/src/datasource/listing/url.rs

@@ -18,7 +18,6 @@
 use crate::execution::context::SessionState;
 use datafusion_common::{DataFusionError, Result};
 use datafusion_execution::object_store::ObjectStoreUrl;
-use datafusion_optimizer::OptimizerConfig;
 use futures::stream::BoxStream;
 use futures::{StreamExt, TryStreamExt};
 use glob::Pattern;
@@ -234,7 +233,7 @@ impl ListingTableUrl {
         store: &'a dyn ObjectStore,
         file_extension: &'a str,
     ) -> Result<BoxStream<'a, Result<ObjectMeta>>> {
-        let exec_options = &ctx.options().execution;
+        let exec_options = &ctx.config_options().execution;
         let ignore_subdirectory = exec_options.listing_table_ignore_subdirectory;
         // If the prefix is a file, use a head request, otherwise list
         let list = match self.is_collection() {

datafusion/core/src/datasource/memory.rs

@@ -238,6 +238,7 @@ impl TableProvider for MemTable {
                         sort_exprs,
                         &df_schema,
                         state.execution_props(),
+                        state.config_options(),
                     )
                 })
                 .collect::<Result<Vec<_>>>()?;

datafusion/core/src/datasource/physical_plan/file_scan_config.rs

@@ -617,10 +617,10 @@ fn create_output_array(
 
 #[cfg(test)]
 mod tests {
-    use arrow_array::Int32Array;
-
     use super::*;
     use crate::{test::columns, test_util::aggr_test_schema};
+    use arrow_array::Int32Array;
+    use datafusion_common::config::ConfigOptions;
 
     #[test]
     fn physical_plan_config_no_projection() {
@@ -1121,6 +1121,7 @@ mod tests {
                             &expr,
                             &DFSchema::try_from(table_schema.as_ref().clone())?,
                             &ExecutionProps::default(),
+                            &ConfigOptions::default(),
                         )
                     })
                     .collect::<Result<Vec<_>>>()?,

datafusion/core/src/execution/session_state.rs

@@ -648,7 +648,7 @@ impl SessionState {
             // analyze & capture output of each rule
             let analyzer_result = self.analyzer.execute_and_check(
                 e.plan.as_ref().clone(),
-                self.options(),
+                self.config_options(),
                 |analyzed_plan, analyzer| {
                     let analyzer_name = analyzer.name().to_string();
                     let plan_type = PlanType::AnalyzedLogicalPlan { analyzer_name };
@@ -708,7 +708,7 @@ impl SessionState {
         } else {
             let analyzed_plan = self.analyzer.execute_and_check(
                 plan.clone(),
-                self.options(),
+                self.config_options(),
                 |_, _| {},
             )?;
             self.optimizer.optimize(analyzed_plan, self, |_, _| {})
@@ -760,13 +760,19 @@ impl SessionState {
         let mut expr = simplifier.coerce(expr, df_schema)?;
 
         // rewrite Exprs to functions if necessary
-        let config_options = self.config_options();
         for rewrite in self.analyzer.function_rewrites() {
             expr = expr
-                .transform_up(|expr| rewrite.rewrite(expr, df_schema, config_options))?
+                .transform_up(|expr| {
+                    rewrite.rewrite(expr, df_schema, self.config_options())
+                })?
                 .data;
         }
-        create_physical_expr(&expr, df_schema, self.execution_props())
+        create_physical_expr(
+            &expr,
+            df_schema,
+            self.execution_props(),
+            self.config_options(),
+        )
     }
 
     /// Return the session ID
@@ -1968,6 +1974,10 @@ impl SimplifyInfo for SessionSimplifyProvider<'_> {
         self.state.execution_props()
     }
 
+    fn config_options(&self) -> &ConfigOptions {
+        self.state.config_options()
+    }
+
     fn get_data_type(&self, expr: &Expr) -> datafusion_common::Result<DataType> {
         expr.get_type(self.df_schema)
     }

datafusion/core/src/physical_optimizer/projection_pushdown.rs

@@ -1421,6 +1421,7 @@ mod tests {
                     )),
                 ],
                 DataType::Int32,
+                Arc::new(ConfigOptions::default()),
             )),
             Arc::new(CaseExpr::try_new(
                 Some(Arc::new(Column::new("d", 2))),
@@ -1486,6 +1487,7 @@ mod tests {
                     )),
                 ],
                 DataType::Int32,
+                Arc::new(ConfigOptions::default()),
             )),
             Arc::new(CaseExpr::try_new(
                 Some(Arc::new(Column::new("d", 3))),
@@ -1554,6 +1556,7 @@ mod tests {
                     )),
                 ],
                 DataType::Int32,
+                Arc::new(ConfigOptions::default()),
             )),
             Arc::new(CaseExpr::try_new(
                 Some(Arc::new(Column::new("d", 2))),
@@ -1619,6 +1622,7 @@ mod tests {
                     )),
                 ],
                 DataType::Int32,
+                Arc::new(ConfigOptions::default()),
             )),
             Arc::new(CaseExpr::try_new(
                 Some(Arc::new(Column::new("d_new", 3))),