From 327fd9f55c4f4828ccdf1e276941ebbb19d93c5c Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Sun, 14 Jul 2024 11:37:26 -0600 Subject: [PATCH] Spark: Avoid attempt to parsing partition path into different types if type inference is disabled --- .../org/apache/spark/sql/delta/util/PartitionUtils.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala index 1dc56b8173b..a5b5d36f0de 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala @@ -528,7 +528,8 @@ private[delta] object PartitionUtils { timeZone: TimeZone, dateFormatter: DateFormatter, timestampFormatter: TimestampFormatter): Literal = { - val decimalTry = Try { + + def decimalTry = Try { // `BigDecimal` conversion can fail when the `field` is not a form of number. val bigDecimal = new JBigDecimal(raw) // It reduces the cases for decimals by disallowing values having scale (eg. `1.1`). @@ -539,7 +540,7 @@ private[delta] object PartitionUtils { Literal(bigDecimal) } - val dateTry = Try { + def dateTry = Try { // try and parse the date, if no exception occurs this is a candidate to be resolved as // DateType dateFormatter.parse(raw) @@ -555,7 +556,7 @@ private[delta] object PartitionUtils { Literal.create(dateValue, DateType) } - val timestampTry = Try { + def timestampTry = Try { val unescapedRaw = unescapePathName(raw) // try and parse the date, if no exception occurs this is a candidate to be resolved as // TimestampType