diff --git a/R/aes.R b/R/aes.R
index 045d388d8a..5e53443988 100644
--- a/R/aes.R
+++ b/R/aes.R
@@ -360,7 +360,7 @@ aes_all <- function(vars) {
   # refer to the data mask
   structure(
     lapply(vars, function(x) new_quosure(as.name(x), emptyenv())),
-    class = "uneval"
+    class = c("unlabelled_uneval", "uneval")
   )
 }
 
diff --git a/R/labels.R b/R/labels.R
index a736e2bf54..27c1e96de6 100644
--- a/R/labels.R
+++ b/R/labels.R
@@ -24,10 +24,16 @@ setup_plot_labels <- function(plot, layers, data) {
   # Find labels from every layer
   for (i in seq_along(layers)) {
     layer <- layers[[i]]
-    exclude <- names(layer$aes_params)
+
     mapping <- layer$computed_mapping
+    if (inherits(mapping, "unlabelled_uneval")) {
+      next
+    }
+
     mapping <- strip_stage(mapping)
     mapping <- strip_dots(mapping, strip_pronoun = TRUE)
+
+    exclude <- names(layer$aes_params)
     mapping <- mapping[setdiff(names(mapping), exclude)]
 
     # Acquire default labels
diff --git a/R/layer.R b/R/layer.R
index 6be74b5d72..4cf2e09417 100644
--- a/R/layer.R
+++ b/R/layer.R
@@ -210,6 +210,8 @@ validate_mapping <- function(mapping, call = caller_env()) {
     }
 
     cli::cli_abort(msg, call = call)
+  } else {
+    return(mapping)
   }
 
   # For backward compatibility with pre-tidy-eval layers
diff --git a/tests/testthat/test-labels.R b/tests/testthat/test-labels.R
index 172eca6364..4befce8af6 100644
--- a/tests/testthat/test-labels.R
+++ b/tests/testthat/test-labels.R
@@ -83,6 +83,13 @@ test_that("Labels from static aesthetics are ignored (#6003)", {
   expect_null(get_labs(p)$colour)
 })
 
+test_that("Labels from annotations are ignored (#6316)", {
+  df <- data.frame(a = 1, b = 2)
+  p <- ggplot(df, aes(a, b)) + annotate("point", x = 1, y = 2) + geom_point()
+  labs <- get_labs(p)
+  expect_equal(labs[c("x", "y")], list(x = "a", y = "b"))
+})
+
 test_that("alt text is returned", {
   p <- ggplot(mtcars, aes(mpg, disp)) +
     geom_point()