diff --git a/NEWS.md b/NEWS.md
index cd4ed0771d..3755e43011 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -148,6 +148,8 @@
   (@teunbrand, #5938, #4327).
 * Fixed bug where empty discrete scales weren't recognised as such 
   (@teunbrand, #5945).
+* (internal) The summary function of `stat_summary()` and `stat_summary_bin()` 
+  is setup once in total instead of once per group (@teunbrand, #5971)
 
 # ggplot2 3.5.1
 
diff --git a/R/stat-summary-bin.R b/R/stat-summary-bin.R
index e9cd675e09..3a4fea585e 100644
--- a/R/stat-summary-bin.R
+++ b/R/stat-summary-bin.R
@@ -64,24 +64,28 @@ stat_summary_bin <- function(mapping = NULL, data = NULL,
 StatSummaryBin <- ggproto("StatSummaryBin", Stat,
   required_aes = c("x", "y"),
 
-  extra_params = c("na.rm", "orientation"),
+  extra_params = c("na.rm", "orientation", "fun.data", "fun.max", "fun.min", "fun.args"),
+
   setup_params = function(data, params) {
-    params$flipped_aes <- has_flipped_aes(data, params, ambiguous = TRUE)
+    params$flipped_aes <- has_flipped_aes(data, params)
+    params$fun <- make_summary_fun(
+      params$fun.data, params$fun,
+      params$fun.max, params$fun.min,
+      params$fun.args %||% list()
+    )
     params
   },
 
-  compute_group = function(data, scales, fun.data = NULL, fun = NULL,
-                           fun.max = NULL, fun.min = NULL, fun.args = list(),
+  compute_group = function(data, scales, fun = NULL,
                            bins = 30, binwidth = NULL, breaks = NULL,
                            origin = NULL, right = FALSE, na.rm = FALSE,
                            flipped_aes = FALSE) {
     data <- flip_data(data, flipped_aes)
-    fun <- make_summary_fun(fun.data, fun, fun.max, fun.min, fun.args)
     x <- flipped_names(flipped_aes)$x
     breaks <- bin2d_breaks(scales[[x]], breaks, origin, binwidth, bins, right = right)
 
     data$bin <- cut(data$x, breaks, include.lowest = TRUE, labels = FALSE)
-    out <- dapply(data, "bin", fun)
+    out <- dapply(data, "bin", fun %||% function(df) mean_se(df$y))
 
     locs <- bin_loc(breaks, out$bin)
     out$x <- locs$mid
diff --git a/R/stat-summary.R b/R/stat-summary.R
index cf2e2ef4c7..6476021fc5 100644
--- a/R/stat-summary.R
+++ b/R/stat-summary.R
@@ -181,18 +181,22 @@ stat_summary <- function(mapping = NULL, data = NULL,
 StatSummary <- ggproto("StatSummary", Stat,
   required_aes = c("x", "y"),
 
-  extra_params = c("na.rm", "orientation"),
+  extra_params = c("na.rm", "orientation", "fun.data", "fun.max", "fun.min", "fun.args"),
+
   setup_params = function(data, params) {
     params$flipped_aes <- has_flipped_aes(data, params)
+    params$fun <- make_summary_fun(
+      params$fun.data, params$fun,
+      params$fun.max, params$fun.min,
+      params$fun.args %||% list()
+    )
     params
   },
 
-  compute_panel = function(data, scales, fun.data = NULL, fun = NULL,
-                     fun.max = NULL, fun.min = NULL, fun.args = list(),
-                     na.rm = FALSE, flipped_aes = FALSE) {
+  compute_panel = function(data, scales, fun = NULL,
+                           na.rm = FALSE, flipped_aes = FALSE) {
     data <- flip_data(data, flipped_aes)
-    fun <- make_summary_fun(fun.data, fun, fun.max, fun.min, fun.args)
-    summarised <- summarise_by_x(data, fun)
+    summarised <- summarise_by_x(data, fun %||% function(df) mean_se(df$y))
     summarised$flipped_aes <- flipped_aes
     flip_data(summarised, flipped_aes)
   }