diff --git a/NEWS.md b/NEWS.md
index 54453eba92..6579c613b1 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,7 @@
 # ggplot2 (development version)
 
+* `position_stack()` now works fully with `geom_text()` (@thomasp85, #4367)
+
 * `geom_tile()` now correctly recognises missing data in `xmin`, `xmax`, `ymin`,
   and `ymax` (@thomasp85 and @sigmapi, #4495)
 
diff --git a/R/position-stack.r b/R/position-stack.r
index 98ad6e5db1..565d311e46 100644
--- a/R/position-stack.r
+++ b/R/position-stack.r
@@ -165,7 +165,7 @@ PositionStack <- ggproto("PositionStack", Position,
 
     data$ymax <- switch(params$var,
       y = data$y,
-      ymax = ifelse(data$ymax == 0, data$ymin, data$ymax)
+      ymax = as.numeric(ifelse(data$ymax == 0, data$ymin, data$ymax))
     )
 
     data <- remove_missing(
@@ -225,8 +225,8 @@ pos_stack <- function(df, width, vjust = 1, fill = FALSE) {
   ymin <- pmin(heights[-n], heights[-1])
   ymax <- pmax(heights[-n], heights[-1])
   df$y <- (1 - vjust) * ymin + vjust * ymax
-  df$ymin <- ifelse(max_is_lower, ymax, ymin)
-  df$ymax <- ifelse(max_is_lower, ymin, ymax)
+  df$ymin <- as.numeric(ifelse(max_is_lower, ymax, ymin))
+  df$ymax <- as.numeric(ifelse(max_is_lower, ymin, ymax))
   df
 }