diff --git a/R/labels.R b/R/labels.R
index 3012868874..2bc5551034 100644
--- a/R/labels.R
+++ b/R/labels.R
@@ -24,9 +24,11 @@ setup_plot_labels <- function(plot, layers, data) {
   # Find labels from every layer
   for (i in seq_along(layers)) {
     layer <- layers[[i]]
+    exclude <- names(layer$aes_params)
     mapping <- layer$computed_mapping
     mapping <- strip_stage(mapping)
     mapping <- strip_dots(mapping, strip_pronoun = TRUE)
+    mapping <- mapping[setdiff(names(mapping), exclude)]
 
     # Acquire default labels
     mapping_default <- make_labels(mapping)
diff --git a/tests/testthat/test-labels.R b/tests/testthat/test-labels.R
index e338226bd4..c659e39cf5 100644
--- a/tests/testthat/test-labels.R
+++ b/tests/testthat/test-labels.R
@@ -74,6 +74,21 @@ test_that("Labels can be extracted from attributes", {
   expect_equal(labels$y, "disp")
 })
 
+test_that("Labels from static aesthetics are ignored (#6003)", {
+
+  df <- data.frame(x = 1, y = 1, f = 1)
+
+  p <- ggplot(df, aes(x, y, colour = f)) + geom_point()
+  labels <- ggplot_build(p)$plot$labels
+
+  expect_equal(labels$colour, "f")
+
+  p <- ggplot(df, aes(x, y, colour = f)) + geom_point(colour = "blue")
+  labels <- ggplot_build(p)$plot$labels
+
+  expect_null(labels$colour)
+})
+
 test_that("alt text is returned", {
   p <- ggplot(mtcars, aes(mpg, disp)) +
     geom_point()