Some Emoji no longer match after 6.1.3

[roderickhsiao]

Thanks for providing the library, we notice that some emoji no longer match the regex after the latest version publish.

Not sure if it because the unicode spec changes? http://www.unicode.org/reports/tr51/

Test case

const emojiRegex = require('emoji-regex');

const emojis = '😁,😂,😃,😄,😅,😆,😉,😊,😋,😌,😍,😏,😒,😓,😔,😖,😘,😚,😜,😝,😞,😠,😡,😢,😣,😤,😥,😨,😩,😪,😫,😭,😰,😱,😲,😳,😵,😷,😸,😹,😺,😻,😼,😽,😾,😿,🙀,🙅,🙆,🙇,🙈,🙉,🙊,🙋,🙌,🙍,🙎,🙏,✂,✅,✈,✉,✊,✋,✌,✏,✒,✔,✖,✨,✳,✴,❄,❇,❌,❎,❓,❔,❕,❗,❤,➕,➖,➗,➡,➰,🚀,🚃,🚄,🚅,🚇,🚉,🚌,🚏,🚑,🚒,🚓,🚕,🚗,🚙,🚚,🚢,🚤,🚥,🚧,🚨,🚩,🚪,🚫,🚬,🚭,🚲,🚶,🚹,🚺,🚻,🚼,🚽,🚾,🛀,Ⓜ,🅰,🅱,🅾,🅿,🆎,🆑,🆒,🆓,🆔,🆕,🆖,🆗,🆘,🆙,🆚,🇩🇪,🇬🇧,🇨🇳,🇯🇵,🇰🇷,🇫🇷,🇪🇸,🇮🇹,🇺🇸,🇷🇺,🈁,🈂,🈚,🈯,🈲,🈳,🈴,🈵,🈶,🈷,🈸,🈹,🈺,🉐,🉑,©,®,‼,⁉,8⃣,9⃣,7⃣,6⃣,1⃣,0⃣,2⃣,3⃣,5⃣,4⃣,#⃣,™,ℹ,↔,↕,↖,↗,↘,↙,↩,↪,⌚,⌛,⏩,⏪,⏫,⏬,⏰,⏳,▪,▫,▶,◀,◻,◼,◽,◾,☀,☁,☎,☑,☔,☕,☝,☺,♈,♉,♊,♋,♌,♍,♎,♏,♐,♑,♒,♓,♠,♣,♥,♦,♨,♻,♿,⚓,⚠,⚡,⚪,⚫,⚽,⚾,⛄,⛅,⛎,⛔,⛪,⛲,⛳,⛵,⛺,⛽,⤴,⤵,⬅,⬆,⬇,⬛,⬜,⭐,⭕,〰,〽,㊗,㊙,🀄,🃏,🌀,🌁,🌂,🌃,🌄,🌅,🌆,🌇,🌈,🌉,🌊,🌋,🌌,🌏,🌑,🌓,🌔,🌕,🌙,🌛,🌟,🌠,🌰,🌱,🌴,🌵,🌷,🌸,🌹,🌺,🌻,🌼,🌽,🌾,🌿,🍀,🍁,🍂,🍃,🍄,🍅,🍆,🍇,🍈,🍉,🍊,🍌,🍍,🍎,🍏,🍑,🍒,🍓,🍔,🍕,🍖,🍗,🍘,🍙,🍚,🍛,🍜,🍝,🍞,🍟,🍠,🍡,🍢,🍣,🍤,🍥,🍦,🍧,🍨,🍩,🍪,🍫,🍬,🍭,🍮,🍯,🍰,🍱,🍲,🍳,🍴,🍵,🍶,🍷,🍸,🍹,🍺,🍻,🎀,🎁,🎂,🎃,🎄,🎅,🎆,🎇,🎈,🎉,🎊,🎋,🎌,🎍,🎎,🎏,🎐,🎑,🎒,🎓,🎠,🎡,🎢,🎣,🎤,🎥,🎦,🎧,🎨,🎩,🎪,🎫,🎬,🎭,🎮,🎯,🎰,🎱,🎲,🎳,🎴,🎵,🎶,🎷,🎸,🎹,🎺,🎻,🎼,🎽,🎾,🎿,🏀,🏁,🏂,🏃,🏄,🏆,🏈,🏊,🏠,🏡,🏢,🏣,🏥,🏦,🏧,🏨,🏩,🏪,🏫,🏬,🏭,🏮,🏯,🏰,🐌,🐍,🐎,🐑,🐒,🐔,🐗,🐘,🐙,🐚,🐛,🐜,🐝,🐞,🐟,🐠,🐡,🐢,🐣,🐤,🐥,🐦,🐧,🐨,🐩,🐫,🐬,🐭,🐮,🐯,🐰,🐱,🐲,🐳,🐴,🐵,🐶,🐷,🐸,🐹,🐺,🐻,🐼,🐽,🐾,👀,👂,👃,👄,👅,👆,👇,👈,👉,👊,👋,👌,👍,👎,👏,👐,👑,👒,👓,👔,👕,👖,👗,👘,👙,👚,👛,👜,👝,👞,👟,👠,👡,👢,👣,👤,👦,👧,👨,👩,👪,👫,👮,👯,👰,👱,👲,👳,👴,👵,👶,👷,👸,👹,👺,👻,👼,👽,👾,👿,💀,💁,💂,💃,💄,💅,💆,💇,💈,💉,💊,💋,💌,💍,💎,💏,💐,💑,💒,💓,💔,💕,💖,💗,💘,💙,💚,💛,💜,💝,💞,💟,💠,💡,💢,💣,💤,💥,💦,💧,💨,💩,💪,💫,💬,💮,💯,💰,💱,💲,💳,💴,💵,💸,💹,💺,💻,💼,💽,💾,💿,📀,📁,📂,📃,📄,📅,📆,📇,📈,📉,📊,📋,📌,📍,📎,📏,📐,📑,📒,📓,📔,📕,📖,📗,📘,📙,📚,📛,📜,📝,📞,📟,📠,📡,📢,📣,📤,📥,📦,📧,📨,📩,📪,📫,📮,📰,📱,📲,📳,📴,📶,📷,📹,📺,📻,📼,🔃,🔊,🔋,🔌,🔍,🔎,🔏,🔐,🔑,🔒,🔓,🔔,🔖,🔗,🔘,🔙,🔚,🔛,🔜,🔝,🔞,🔟,🔠,🔡,🔢,🔣,🔤,🔥,🔦,🔧,🔨,🔩,🔪,🔫,🔮,🔯,🔰,🔱,🔲,🔳,🔴,🔵,🔶,🔷,🔸,🔹,🔺,🔻,🔼,🔽,🕐,🕑,🕒,🕓,🕔,🕕,🕖,🕗,🕘,🕙,🕚,🕛,🗻,🗼,🗽,🗾,🗿,😀,😇,😈,😎,😐,😑,😕,😗,😙,😛,😟,😦,😧,😬,😮,😯,😴,😶,🚁,🚂,🚆,🚈,🚊,🚍,🚎,🚐,🚔,🚖,🚘,🚛,🚜,🚝,🚞,🚟,🚠,🚡,🚣,🚦,🚮,🚯,🚰,🚱,🚳,🚴,🚵,🚷,🚸,🚿,🛁,🛂,🛃,🛄,🛅,🌍,🌎,🌐,🌒,🌖,🌗,🌘,🌚,🌜,🌝,🌞,🌲,🌳,🍋,🍐,🍼,🏇,🏉,🏤,🐀,🐁,🐂,🐃,🐄,🐅,🐆,🐇,🐈,🐉,🐊,🐋,🐏,🐐,🐓,🐕,🐖,🐪,👥,👬,👭,💭,💶,💷,📬,📭,📯,📵,🔀,🔁,🔂,🔄,🔅,🔆,🔇,🔉,🔕,🔬,🔭,🕜,🕝,🕞,🕟,🕠,🕡,🕢,🕣,🕤,🕥,🕦,🕧'.split(',');

const exception = [];
emojis.forEach((emoji) => {
  const match = emojiRegex().exec(emoji);
  if (!match) { exception.push(emoji) }
});
console.log('Exception length', exception.length);
console.log(JSON.stringify(exception));

6.1.0

Exception length 0

[]

6.1.3

Exception length 72

["✂","✈","✉","✏","✒","✔","✖","✳","✴","❄","❇","❤","➡","Ⓜ","🅰","🅱","🅾","🅿","🈂","🈷","©","®","‼","⁉","8⃣","9⃣","7⃣","6⃣","1⃣","0⃣","2⃣","3⃣","5⃣","4⃣","#⃣","™","ℹ","↔","↕","↖","↗","↘","↙","↩","↪","▪","▫","▶","◀","◻","◼","☀","☁","☎","☑","☺","♠","♣","♥","♦","♨","♻","⚠","⤴","⤵","⬅","⬆","⬇","〰","〽","㊗","㊙"]

---

Using https://github.com/Kikobeats/emojis-list as spec

6.1.0

Exception length 118

["🇦","🇧","🇨","🇩","🇪","🇫","🇬","🇭","🇮","🇯","🇰","🇱","🇲","🇳","🇴","🇵","🇶","🇷","🇸","🇹","🇺🇳","🇺","🇻","🇼","🇽","🇾","🇿","🕺","🖤","🗨","🛑","🛒","🛴","🛵","🛶","🤙","🤚","🤛","🤜","🤝","🤞","🤠","🤡","🤢","🤣","🤤","🤥","🤦‍♀️","🤦‍♂️","🤦","🤧","🤰","🤳","🤴","🤵","🤶","🤷‍♀️","🤷‍♂️","🤷","🤸‍♀️","🤸‍♂️","🤸","🤹‍♀️","🤹‍♂️","🤹","🤺","🤼‍♀️","🤼‍♂️","🤼","🤽‍♀️","🤽‍♂️","🤽","🤾‍♀️","🤾‍♂️","🤾","🥀","🥁","🥂","🥃","🥄","🥅","🥇","🥈","🥉","🥊","🥋","🥐","🥑","🥒","🥓","🥔","🥕","🥖","🥗","🥘","🥙","🥚","🥛","🥜","🥝","🥞","🦅","🦆","🦇","🦈","🦉","🦊","🦋","🦌","🦍","🦎","🦏","🦐","🦑","♀","♂","⚕",""]

6.1.3

Exception length 209

["🅰","🅱","🅾","🅿","🈂","🈷","🌡","🌤","🌥","🌦","🌧","🌨","🌩","🌪","🌫","🌬","🌶","🍽","🎖","🎗","🎙","🎚","🎛","🎞","🎟","🏍","🏎","🏔","🏕","🏖","🏗","🏘","🏙","🏚","🏛","🏜","🏝","🏞","🏟","🏳","🏵","🏷","🐿","👁‍🗨","👁","📽","🕉","🕊","🕯","🕰","🕳","🕶","🕷","🕸","🕹","🖇","🖊","🖋","🖌","🖍","🖥","🖨","🖱","🖲","🖼","🗂","🗃","🗄","🗑","🗒","🗓","🗜","🗝","🗞","🗡","🗣","🗨","🗯","🗳","🗺","🛋","🛍","🛎","🛏","🛠","🛡","🛢","🛣","🛤","🛥","🛩","🛰","🛳","‼","⁉","™","ℹ","↔","↕","↖","↗","↘","↙","↩","↪","#⃣","⌨","⏏","⏭","⏮","⏯","⏱","⏲","⏸","⏹","⏺","Ⓜ","▪","▫","▶","◀","◻","◼","☀","☁","☂","☃","☄","☎","☑","☘","☠","☢","☣","☦","☪","☮","☯","☸","☹","☺","♀","♂","♠","♣","♥","♦","♨","♻","⚒","⚔","⚕","⚖","⚗","⚙","⚛","⚜","⚠","⚰","⚱","⛈","⛏","⛑","⛓","⛩","⛰","⛱","⛴","⛷","⛸","✂","✈","✉","✏","✒","✔","✖","✝","✡","✳","✴","❄","❇","❣","❤","➡","⤴","⤵","*⃣","⬅","⬆","⬇","0⃣","〰","〽","1⃣","2⃣","㊗","㊙","3⃣","4⃣","5⃣","6⃣","7⃣","8⃣","9⃣","©","®",""]

[mathiasbynens]

Support for sequences is not implemented yet since the rewrite. 37d8faac5a725e37bb5e9ff9531dd9b241c63ae0

[paulirish]

Here's the test string that me and @notwaldorf are using for testing an emoji-extraction script:

👩🏿😎🙈🏳️‍🌈👨‍👨‍👧‍👦

eager for sequence support. :)

[mathiasbynens]

I’ve updated unicode-tr51 so it now exports a list of all emoji sequences as an array of strings: https://github.com/mathiasbynens/unicode-tr51/blob/v8.1.1/sequences.js

Now we can pass that to regexgen to generate a compact regex for the sequences, and combine that with the existing regex. I’ve written the code to do this already, but ran into (what I think is) a regexgen bug: https://github.com/devongovett/regexgen/issues/10 Once that’s fixed, emoji-regex gets sequence support!

[paulirish]

@mathiasbynens you are such a pro. :)

[mathiasbynens]

😊

[roderickhsiao]

Definitely a bravo for the lightening fix 💯

[roderickhsiao]

@mathiasbynens thanks for the quick fix.

I tried 6.2.0 using the same test cases, still getting the same results. probably a build/publish issue?

[mathiasbynens]

Ah, I see. The problematic test cases, e.g. U+1F321 THERMOMETER 🌡️ are not Emoji_Presentation characters. This means that they get rendered as text by default (according to the Unicode data files), and only get rendered as an emoji when followed by U+FE0F VARIATION SELECTOR-16. (more info)

From http://unicode.org/Public/emoji/5.0/emoji-test.txt:

1F321 FE0F                                 ; fully-qualified     # 🌡️ thermometer
1F321                                      ; non-fully-qualified # 🌡 thermometer

(On my system U+1F321 gets rendered as an emoji even without the variation selector, but of course that’s something that differs anyway depending on the fonts you’re using etc. This package is purely based on the Unicode data.)

I don’t really see how to move forward here — should emoji-regex expose a secondary regex that matches text-emoji such as 🌡️ as well?

[devongovett]

As an alternative if you want to exactly match a particular font, there is apple-color-emoji which generates a regex from the actual Mac emoji font. Perhaps the same could be done for other fonts.

[roderickhsiao]

Good find @mathiasbynens

Probably an config like emojiRegex({config}) to return regex including non-fully-qualified list? But then we are exploding the file size if people are not using it. Or as yo mentioned, a secondary regex export?

[mathiasbynens]

@roderickhsiao Please try v6.3.0 + the last paragraph in https://github.com/mathiasbynens/emoji-regex#installation and let me know if that solves your use case!

[roderickhsiao]

it works perfectly, probably in readme we should import emojiRegex from 'emoji-regex/dist/text'; as we are not exposing dist directly :)

Thanks for quick fix!

[mathiasbynens]

@roderickhsiao Thanks! Fixed in v6.4.0 by removing the dist folder.

[roderickhsiao]

Work like a charm, thanks!

[devongovett]

regexgen fix for the bug mentioned above is here: https://github.com/devongovett/regexgen/pull/14. Released in v1.2.3. 🎉