diff --git a/book/src/configuration.md b/book/src/configuration.md
index 7f2a4acfb..8048f5484 100644
--- a/book/src/configuration.md
+++ b/book/src/configuration.md
@@ -34,7 +34,7 @@ hidden = false
 | `middle-click-paste` | Middle click paste support. | `true` |
 | `scroll-lines` | Number of lines to scroll per scroll wheel step. | `3` |
 | `shell` | Shell to use when running external commands. | Unix: `["sh", "-c"]`<br/>Windows: `["cmd", "/C"]` |
-| `line-number` | Line number display (`absolute`, `relative`) | `absolute` |
+| `line-number` | Line number display: `absolute` simply shows each line's number, while `relative` shows the distance from the current line. When unfocused or in insert mode, `relative` will still show absolute line numbers. | `absolute` |
 | `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
 | `auto-pairs` | Enable automatic insertion of pairs to parenthese, brackets, etc. | `true` |
 | `auto-completion` | Enable automatic pop up of auto-completion. | `true` |
diff --git a/helix-view/src/editor.rs b/helix-view/src/editor.rs
index 82ef0cdc4..aa2df6f74 100644
--- a/helix-view/src/editor.rs
+++ b/helix-view/src/editor.rs
@@ -170,7 +170,9 @@ impl Default for CursorShapeConfig {
 pub enum LineNumber {
     /// Show absolute line number
     Absolute,
-    /// Show relative line number to the primary cursor
+
+    /// If focused and in normal/select mode, show relative line number to the primary cursor.
+    /// If unfocused or in insert mode, show absolute line number.
     Relative,
 }
 
diff --git a/helix-view/src/gutter.rs b/helix-view/src/gutter.rs
index af016c56e..113da642e 100644
--- a/helix-view/src/gutter.rs
+++ b/helix-view/src/gutter.rs
@@ -58,29 +58,31 @@ pub fn line_number<'doc>(
         .char_to_line(doc.selection(view.id).primary().cursor(text));
 
     let config = config.line_number;
+    let mode = doc.mode;
 
     Box::new(move |line: usize, selected: bool, out: &mut String| {
         if line == last_line && !draw_last {
             write!(out, "{:>1$}", '~', width).unwrap();
             Some(linenr)
         } else {
-            use crate::editor::LineNumber;
-            let line = match config {
-                LineNumber::Absolute => line + 1,
-                LineNumber::Relative => {
-                    if current_line == line {
-                        line + 1
-                    } else {
-                        abs_diff(current_line, line)
-                    }
-                }
+            use crate::{document::Mode, editor::LineNumber};
+
+            let relative = config == LineNumber::Relative
+                && mode != Mode::Insert
+                && is_focused
+                && current_line != line;
+
+            let display_num = if relative {
+                abs_diff(current_line, line)
+            } else {
+                line + 1
             };
             let style = if selected && is_focused {
                 linenr_select
             } else {
                 linenr
             };
-            write!(out, "{:>1$}", line, width).unwrap();
+            write!(out, "{:>1$}", display_num, width).unwrap();
             Some(style)
         }
     })