Fix and improve handling of Skin Tone Modifiers:

- Fix that modifiers were ignored when not part of a larger sequence #29
- Only check for valid base characters when in Emoji level is RGI
- Improve docs and specs

Author: janlelis

CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 3.1.4
+
+- Fix that skin tone modifiers were ignored when used in a non-ZWJ sequence
+  context (= single emoji char + modifier) #29
+- Add more docs and specs about modifier handling
+
 ## 3.1.3
 
 Better handling of non-UTF-8 strings, patch by @Earlopain:

README.md

@@ -101,12 +101,16 @@ There are many Emoji which get constructed by combining other Emoji in a sequenc
 
 Another aspect where terminals disagree is whether Emoji characters which have a text presentation by default (width 1) should be turned into full-width (width 2) when combined with Variation Selector 16 (*U+FEOF*).
 
+Finally, it varies if Skin Tone Modifiers can be applied to all characters or just to those with the "Emoji Base" property.
+
 Emoji Type  | Width / Comment
 ------------|----------------
-Basic/Single Emoji character without Variation Selector | No special handling
-Basic/Single Emoji character with VS15 (Text)           | No special handling
-Basic/Single Emoji character with VS16 (Emoji)          | 2 or East Asian Width (see table below)
-Emoji Sequence                                          | 2 if Emoji belongs to configured Emoji set (see table below)
+Basic/Single Emoji character without Variation Selector   | No special handling
+Basic/Single Emoji character with VS15 (Text)             | No special handling
+Basic/Single Emoji character with VS16 (Emoji)            | 2 or East Asian Width (see table below)
+Single Emoji character with Skin Tone Modifier            | 2
+Skin Tone Modifier used in isolation or with invalid base | 2 if Emoji mode is configured to RGI
+Emoji Sequence                                            | 2 if Emoji belongs to configured Emoji set (see table below)
 
 #### Emoji Modes
 

lib/unicode/display_width.rb

@@ -42,7 +42,9 @@ class DisplayWidth
       ),
       Unicode::Emoji::REGEX_EMOJI_KEYCAP
     )
-    REGEX_EMOJI_ALL_SEQUENCES = Regexp.union(/.[\u{1F3FB}-\u{1F3FF}\u{FE0F}]?(\u{200D}.[\u{1F3FB}-\u{1F3FF}\u{FE0F}]?)+/, Unicode::Emoji::REGEX_EMOJI_KEYCAP)
+
+    # ebase = Unicode::Emoji::REGEX_PROP_MODIFIER_BASE.source
+    REGEX_EMOJI_ALL_SEQUENCES = Regexp.union(/.[\u{1F3FB}-\u{1F3FF}\u{FE0F}]?(\u{200D}.[\u{1F3FB}-\u{1F3FF}\u{FE0F}]?)+|.[\u{1F3FB}-\u{1F3FF}]/, Unicode::Emoji::REGEX_EMOJI_KEYCAP)
     REGEX_EMOJI_ALL_SEQUENCES_AND_VS16 = Regexp.union(REGEX_EMOJI_ALL_SEQUENCES, REGEX_EMOJI_VS16)
 
     # Returns monospace display width of string

misc/terminal-emoji-width.rb

@@ -11,6 +11,14 @@
 puts
 puts RULER + "⛹️" + ABC
 
+puts "1C) BASE EMOJI CHARACTER + MODIFIER"
+puts
+puts RULER + "🏃🏽" + ABC
+
+puts "1D) MODIFIER IN ISOLATION"
+puts
+puts RULER + "Z🏽" + ABC
+
 puts "2) RGI EMOJI SEQ"
 puts
 puts RULER + "🏃🏼‍♀‍➡" + ABC

spec/display_width_spec.rb

@@ -221,10 +221,6 @@
     end
 
     describe '(special emoji / emoji sequences)' do
-      it 'works with singleton skin tone modifiers: width 2' do
-        expect( "🏿".display_width(emoji: :all) ).to eq 2
-      end
-
       it 'works with flags: width 2' do
         expect( "🇵🇹".display_width(emoji: :all) ).to eq 2
       end
@@ -239,8 +235,12 @@
     end
 
     describe '(modifiers and zwj sequences)' do
+      it 'applies simple skin tone modifiers' do
+        expect( "👏🏽".display_width(emoji: :rgi) ).to eq 2
+      end
+
       it 'counts RGI Emoji ZWJ sequence as width 2' do
-        expect( "🤾🏽‍♀️".display_width(1, emoji: :rgi) ).to eq 2
+        expect( "🤾🏽‍♀️".display_width(emoji: :rgi) ).to eq 2
       end
 
       it 'works for emoji involving characters which are east asian ambiguous' do
@@ -253,6 +253,7 @@
         it 'does no Emoji adjustments when emoji suport is disabled' do
           expect( "🤾🏽‍♀️".display_width(emoji: false) ).to eq 5
           expect( "❣️".display_width(emoji: :none) ).to eq 1
+          expect( "👏🏽".display_width(emoji: :none) ).to eq 4
         end
       end
 
@@ -277,6 +278,8 @@
           expect( "🤾🏽‍♀️".display_width(emoji: :rgi) ).to eq 2 # FQE
           expect( "🤾🏽‍♀".display_width(emoji: :rgi) ).to eq 2 # MQE
           expect( "❤‍🩹".display_width(emoji: :rgi) ).to eq 2 # UQE
+          expect( "👏🏽".display_width(emoji: :rgi) ).to eq 2 # Modifier
+          expect( "J🏽".display_width(emoji: :rgi) ).to eq 3 # Modifier with invalid base
           expect( "🤠‍🤢".display_width(emoji: :rgi) ).to eq 4 # Non-RGI/well-formed
           expect( "🚄🏾‍▶️".display_width(emoji: :rgi) ).to eq 6 # Invalid/non-Emoji sequence
         end
@@ -308,6 +311,8 @@
           expect( "🤾🏽‍♀️".display_width(emoji: :possible) ).to eq 2 # FQE
           expect( "🤾🏽‍♀".display_width(emoji: :possible) ).to eq 2 # MQE
           expect( "❤‍🩹".display_width(emoji: :possible) ).to eq 2 # UQE
+          expect( "👏🏽".display_width(emoji: :possible) ).to eq 2 # Modifier
+          expect( "J🏽".display_width(emoji: :possible) ).to eq 3 # Modifier with invalid base
           expect( "🤠‍🤢".display_width(emoji: :possible) ).to eq 2 # Non-RGI/well-formed
           expect( "🚄🏾‍▶️".display_width(emoji: :possible) ).to eq 6 # Invalid/non-Emoji sequence
         end
@@ -322,6 +327,9 @@
           expect( "🤾🏽‍♀️".display_width(emoji: :all) ).to eq 2 # FQE
           expect( "🤾🏽‍♀".display_width(emoji: :all) ).to eq 2 # MQE
           expect( "❤‍🩹".display_width(emoji: :all) ).to eq 2 # UQE
+          expect( "👏🏽".display_width(emoji: :all) ).to eq 2 # Modifier
+          expect( "👏🏽".display_width(emoji: :all) ).to eq 2 # Modifier
+          expect( "J🏽".display_width(emoji: :all) ).to eq 2 # Modifier with invalid base
           expect( "🤠‍🤢".display_width(emoji: :all) ).to eq 2 # Non-RGI/well-formed
           expect( "🚄🏾‍▶️".display_width(emoji: :all) ).to eq 2 # Invalid/non-Emoji sequence
         end
@@ -336,6 +344,8 @@
           expect( "🤾🏽‍♀️".display_width(emoji: :all_no_vs16) ).to eq 2 # FQE
           expect( "🤾🏽‍♀".display_width(emoji: :all_no_vs16) ).to eq 2 # MQE
           expect( "❤‍🩹".display_width(emoji: :all_no_vs16) ).to eq 2 # UQE
+          expect( "👏🏽".display_width(emoji: :all_no_vs16) ).to eq 2 # Modifier
+          expect( "J🏽".display_width(emoji: :all_no_vs16) ).to eq 2 # Modifier with wrong base
           expect( "🤠‍🤢".display_width(emoji: :all_no_vs16) ).to eq 2 # Non-RGI/well-formed
           expect( "🚄🏾‍▶️".display_width(emoji: :all_no_vs16) ).to eq 2 # Invalid/non-Emoji sequence
         end