diff --git a/R/geom-label.R b/R/geom-label.R
index 343a5ae28e..59f76e7cb4 100644
--- a/R/geom-label.R
+++ b/R/geom-label.R
@@ -2,7 +2,8 @@
 #' @rdname geom_text
 #' @param label.padding Amount of padding around label. Defaults to 0.25 lines.
 #' @param label.r Radius of rounded corners. Defaults to 0.15 lines.
-#' @param label.size Size of label border, in mm.
+#' @param label.size `r lifecycle::badge("deprecated")` Please use `linewidth` to set the width of the border.
+#' @param border.colour,border.color Colour of the label's border. If `NULL` (default), it will fall back to the text colour. `border.color` is an alias.
 geom_label <- function(mapping = NULL, data = NULL,
                        stat = "identity", position = "identity",
                        ...,
@@ -11,11 +12,13 @@ geom_label <- function(mapping = NULL, data = NULL,
                        nudge_y = 0,
                        label.padding = unit(0.25, "lines"),
                        label.r = unit(0.15, "lines"),
-                       label.size = 0.25,
+                       label.size = deprecated(),
                        size.unit = "mm",
+                       border.colour = NULL,
                        na.rm = FALSE,
                        show.legend = NA,
-                       inherit.aes = TRUE) {
+                       inherit.aes = TRUE,
+                       border.color = border.colour) {
   if (!missing(nudge_x) || !missing(nudge_y)) {
     if (!missing(position)) {
       cli::cli_abort(c(
@@ -27,6 +30,10 @@ geom_label <- function(mapping = NULL, data = NULL,
     position <- position_nudge(nudge_x, nudge_y)
   }
 
+  if (lifecycle::is_present(label.size)) {
+    deprecate_warn0("3.5.0", "geom_label(label.size)", "geom_label(linewidth)")
+  }
+
   layer(
     data = data,
     mapping = mapping,
@@ -39,8 +46,8 @@ geom_label <- function(mapping = NULL, data = NULL,
       parse = parse,
       label.padding = label.padding,
       label.r = label.r,
-      label.size = label.size,
       size.unit = size.unit,
+      border.colour = border.colour %||% border.color,
       na.rm = na.rm,
       ...
     )
@@ -58,15 +65,15 @@ GeomLabel <- ggproto("GeomLabel", Geom,
   default_aes = aes(
     colour = "black", fill = "white", size = 3.88, angle = 0,
     hjust = 0.5, vjust = 0.5, alpha = NA, family = "", fontface = 1,
-    lineheight = 1.2
+    lineheight = 1.2, linetype = 1, linewidth = 0.25
   ),
 
   draw_panel = function(self, data, panel_params, coord, parse = FALSE,
                         na.rm = FALSE,
                         label.padding = unit(0.25, "lines"),
                         label.r = unit(0.15, "lines"),
-                        label.size = 0.25,
-                        size.unit = "mm") {
+                        size.unit = "mm",
+                        border.colour = NULL) {
     lab <- data$label
     if (parse) {
       lab <- parse_safe(as.character(lab))
@@ -102,9 +109,10 @@ GeomLabel <- ggproto("GeomLabel", Geom,
           lineheight = row$lineheight
         ),
         rect.gp = gpar(
-          col = if (isTRUE(all.equal(label.size, 0))) NA else row$colour,
+          col = ifelse(row$linewidth == 0, NA, border.colour %||% row$colour),
           fill = alpha(row$fill, row$alpha),
-          lwd = label.size * .pt
+          lty = row$linetype,
+          lwd = row$linewidth * .pt
         )
       )
     })
diff --git a/R/geom-text.R b/R/geom-text.R
index 0f4ed1918e..5e17740d16 100644
--- a/R/geom-text.R
+++ b/R/geom-text.R
@@ -89,6 +89,20 @@
 #'   scale_colour_discrete(l = 40)
 #' p + geom_label(aes(fill = factor(cyl)), colour = "white", fontface = "bold")
 #'
+#' # If border.color is NULL or not set, the border will use the text color
+#' p + geom_label(aes(colour = factor(cyl)))
+#'
+#' # Alternatively, border.color can have a static value
+#' p + geom_label(aes(colour = factor(cyl)), border.colour = "black")
+#'
+#' # Use linetype and linewidth aesthetics to add highlights
+#' p + 
+#'  geom_label(
+#'    aes(fill = factor(cyl), linetype = qsec < 15),
+#'    border.colour = "black", colour = "white"
+#'  ) +
+#'  scale_linetype_manual(values = c("solid", "blank"), limits = TRUE)
+#'
 #' p + geom_text(aes(size = wt))
 #' # Scale height of text, rather than sqrt(height)
 #' p +
diff --git a/R/legend-draw.R b/R/legend-draw.R
index 5f8c202f07..ebb3b3bfd7 100644
--- a/R/legend-draw.R
+++ b/R/legend-draw.R
@@ -252,9 +252,29 @@ draw_key_text <- function(data, params, size) {
 #' @export
 #' @rdname draw_key
 draw_key_label <- function(data, params, size) {
-  grobTree(
-    draw_key_rect(data, list()),
-    draw_key_text(data, list())
+  if(is.null(params$label.padding)) params$label.padding <- unit(0.25, "lines")
+  if(length(params$label.padding) == 1) params$label.padding[2:4] <- params$label.padding
+
+  labelGrob(
+    label = data$label %||% "a",
+    x = 0.5,
+    y = 0.5,
+    padding = params$label.padding,
+    r = data$label.r %||% unit(0.1, "snpc"),
+    angle = data$angle %||% 0,
+    text.gp = gpar(
+      col = alpha(data$colour %||% "white", data$alpha),
+      fontfamily = data$family %||% "",
+      fontface = data$fontface %||% 1,
+      fontsize = (data$size %||% 3.88) * .pt
+    ),
+    rect.gp = gpar(
+      col = alpha(params$border.colour %||% params$border.color %||% data$colour %||% "black", data$alpha),
+      fill = alpha(data$fill %||% "white", data$alpha),
+      lty = data$linetype,
+      lwd = data$linewidth * .pt
+    ),
+    vp = NULL
   )
 }
 
diff --git a/man/geom_text.Rd b/man/geom_text.Rd
index f9dfe385a6..fbc3d5a5e7 100644
--- a/man/geom_text.Rd
+++ b/man/geom_text.Rd
@@ -16,11 +16,13 @@ geom_label(
   nudge_y = 0,
   label.padding = unit(0.25, "lines"),
   label.r = unit(0.15, "lines"),
-  label.size = 0.25,
+  label.size = deprecated(),
   size.unit = "mm",
+  border.colour = NULL,
   na.rm = FALSE,
   show.legend = NA,
-  inherit.aes = TRUE
+  inherit.aes = TRUE,
+  border.color = border.colour
 )
 
 geom_text(
@@ -85,12 +87,14 @@ Cannot be jointly specified with \code{position}.}
 
 \item{label.r}{Radius of rounded corners. Defaults to 0.15 lines.}
 
-\item{label.size}{Size of label border, in mm.}
+\item{label.size}{\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#deprecated}{\figure{lifecycle-deprecated.svg}{options: alt='[Deprecated]'}}}{\strong{[Deprecated]}} Please use \code{linewidth} to set the size of the border.}
 
 \item{size.unit}{How the \code{size} aesthetic is interpreted: as millimetres
 (\code{"mm"}, default), points (\code{"pt"}), centimetres (\code{"cm"}), inches (\code{"in"}),
 or picas (\code{"pc"}).}
 
+\item{border.colour}{Colour of the label's border. If \code{NULL}, it will fall back to the text colour.}
+
 \item{na.rm}{If \code{FALSE}, the default, missing values are removed with
 a warning. If \code{TRUE}, missing values are silently removed.}
 
@@ -105,6 +109,8 @@ rather than combining with them. This is most useful for helper functions
 that define both data and aesthetics and shouldn't inherit behaviour from
 the default plot specification, e.g. \code{\link[=borders]{borders()}}.}
 
+\item{border.color}{An alias for \code{border.colour}.}
+
 \item{check_overlap}{If \code{TRUE}, text that overlaps previous text in the
 same layer will not be plotted. \code{check_overlap} happens at draw time and in
 the order of the data. Therefore data should be arranged by the label
@@ -207,6 +213,18 @@ p + geom_text(aes(colour = factor(cyl))) +
   scale_colour_discrete(l = 40)
 p + geom_label(aes(fill = factor(cyl)), colour = "white", fontface = "bold")
 
+# If border.color is NULL or not set, the border will use the text color
+p + geom_label(aes(color = factor(cyl)))
+
+# Alternatively, border.color can have a static value
+p + geom_label(aes(color = factor(cyl)), border.color = "black")
+
+# Use linetype and linewidth aesthetics to add highlights
+p + geom_label(
+  aes(fill = factor(cyl), linetype = qsec < 15),
+  border.colour = "black", color = "white", linewidth = 1) +
+scale_linetype_manual(values=c("solid", "blank"), limits = TRUE, labels = "1/4 mi < 15s", name = NULL)
+
 p + geom_text(aes(size = wt))
 # Scale height of text, rather than sqrt(height)
 p +
diff --git a/man/ggsf.Rd b/man/ggsf.Rd
index fca4f896d3..17389f80e8 100644
--- a/man/ggsf.Rd
+++ b/man/ggsf.Rd
@@ -236,7 +236,7 @@ Cannot be jointly specified with \code{position}.}
 
 \item{label.r}{Radius of rounded corners. Defaults to 0.15 lines.}
 
-\item{label.size}{Size of label border, in mm.}
+\item{label.size}{\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#deprecated}{\figure{lifecycle-deprecated.svg}{options: alt='[Deprecated]'}}}{\strong{[Deprecated]}} Please use \code{linewidth} to set the size of the border.}
 
 \item{fun.geometry}{A function that takes a \code{sfc} object and returns a \code{sfc_POINT} with the
 same length as the input. If \code{NULL}, \code{function(x) sf::st_point_on_surface(sf::st_zm(x))}