keep combining marks with their letter when tokenizing, fixes #1099

src/1-one/tokenize/methods/03-whitespace/tokenize.js

@@ -2,6 +2,9 @@
 
 //we have slightly different rules for start/end - like #hashtags.
 const isLetter = /\p{Letter}/u
+const hasMark = /\p{Mark}/u
+// a letter with its combining marks, or any one char
+const splitChars = /\p{Letter}\p{Mark}*|./gsu
 const isNumber = /[\p{Number}\p{Currency_Symbol}]/u
 const hasAcronym = /^[a-z]\.([a-z]\.)+/i
 const chillin = /[sn]['’]$/
@@ -28,7 +31,8 @@ const normalizePunctuation = function (str, model) {
   const original = str
   let pre = ''
   let post = ''
-  const chars = Array.from(str)
+  // keep combining marks on their letter - 'cafe\u0301'
+  const chars = hasMark.test(str) ? str.match(splitChars) : Array.from(str)
   let start = 0
   let end = chars.length
 

tests/01-one/tokenize/unicode.test.js

@@ -43,3 +43,24 @@ test('fullwidth-ascii', function (t) {
   t.equal(nlp('Ｈｅｌｌｏ ２０２４').text(), 'Ｈｅｌｌｏ ２０２４', here + 'fullwidth text unchanged')
   t.end()
 })
+
+test('combining-marks', function (t) {
+  const arr = [
+    ['caf\u00e9 au lait'.normalize('NFD'), 'cafe\u0301', ' '],
+    ['Poincare\u0301', 'Poincare\u0301', ''],
+    ['E\u0300\u0304 x', 'E\u0300\u0304', ' '],
+    ['(cafe\u0301)', 'cafe\u0301', ')'],
+    ['cafe\u0301!', 'cafe\u0301', '!'],
+    ['\u0928\u092e\u0938\u094d\u0924\u0947 x', '\u0928\u092e\u0938\u094d\u0924\u0947', ' '],
+    ['\u2764\ufe0f x', '\u2764\ufe0f', ' '],
+  ]
+  arr.forEach(a => {
+    const [str, text, post] = a
+    const doc = nlp(str)
+    const term = doc.json()[0].terms[0]
+    t.equal(term.text, text, here + `text "${str}"`)
+    t.equal(term.post, post, here + `post "${str}"`)
+    t.equal(doc.text(), str, here + `identical-text "${str}"`)
+  })
+  t.end()
+})