From 236b97ae91df9c705badcd8ba5aece080967512c Mon Sep 17 00:00:00 2001
From: dualfroz <me@dualfroz.com>
Date: Wed, 14 Oct 2026 19:39:16 +0200
Subject: [PATCH] keep combining marks with their letter when tokenizing, fixes
 #1099

---
 .../methods/03-whitespace/tokenize.js         |  6 +++++-
 tests/01-one/tokenize/unicode.test.js         | 21 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/1-one/tokenize/methods/03-whitespace/tokenize.js b/src/1-one/tokenize/methods/03-whitespace/tokenize.js
index d6a541a..038413d 100644
--- a/src/1-one/tokenize/methods/03-whitespace/tokenize.js
+++ b/src/1-one/tokenize/methods/03-whitespace/tokenize.js
@@ -2,6 +2,9 @@
 
 //we have slightly different rules for start/end - like #hashtags.
 const isLetter = /\p{Letter}/u
+const hasMark = /\p{Mark}/u
+// a letter with its combining marks, or any one char
+const splitChars = /\p{Letter}\p{Mark}*|./gsu
 const isNumber = /[\p{Number}\p{Currency_Symbol}]/u
 const hasAcronym = /^[a-z]\.([a-z]\.)+/i
 const chillin = /[sn]['’]$/
@@ -28,7 +31,8 @@ const normalizePunctuation = function (str, model) {
   const original = str
   let pre = ''
   let post = ''
-  const chars = Array.from(str)
+  // keep combining marks on their letter - 'cafe\u0301'
+  const chars = hasMark.test(str) ? str.match(splitChars) : Array.from(str)
   let start = 0
   let end = chars.length
 
diff --git a/tests/01-one/tokenize/unicode.test.js b/tests/01-one/tokenize/unicode.test.js
index 39af2c5..4b58d2d 100644
--- a/tests/01-one/tokenize/unicode.test.js
+++ b/tests/01-one/tokenize/unicode.test.js
@@ -43,3 +43,24 @@ test('fullwidth-ascii', function (t) {
   t.equal(nlp('Ｈｅｌｌｏ ２０２４').text(), 'Ｈｅｌｌｏ ２０２４', here + 'fullwidth text unchanged')
   t.end()
 })
+
+test('combining-marks', function (t) {
+  const arr = [
+    ['caf\u00e9 au lait'.normalize('NFD'), 'cafe\u0301', ' '],
+    ['Poincare\u0301', 'Poincare\u0301', ''],
+    ['E\u0300\u0304 x', 'E\u0300\u0304', ' '],
+    ['(cafe\u0301)', 'cafe\u0301', ')'],
+    ['cafe\u0301!', 'cafe\u0301', '!'],
+    ['\u0928\u092e\u0938\u094d\u0924\u0947 x', '\u0928\u092e\u0938\u094d\u0924\u0947', ' '],
+    ['\u2764\ufe0f x', '\u2764\ufe0f', ' '],
+  ]
+  arr.forEach(a => {
+    const [str, text, post] = a
+    const doc = nlp(str)
+    const term = doc.json()[0].terms[0]
+    t.equal(term.text, text, here + `text "${str}"`)
+    t.equal(term.post, post, here + `post "${str}"`)
+    t.equal(doc.text(), str, here + `identical-text "${str}"`)
+  })
+  t.end()
+})