From ef0f63b48979fd411959c2f0467a8f6775632fed Mon Sep 17 00:00:00 2001
From: upstream <upstream@localhost>
Date: Fri, 2 Oct 2026 16:50:12 +0200
Subject: [PATCH] Upstream spencermountain/compromise state of the touched
 files before 2bffb43267

---
 .../methods/03-whitespace/tokenize.js         | 88 +++++++++++++++++++
 tests/one/tokenize/punctuation.test.js        | 54 ++++++++++++
 2 files changed, 142 insertions(+)
 create mode 100644 src/1-one/tokenize/methods/03-whitespace/tokenize.js
 create mode 100644 tests/one/tokenize/punctuation.test.js

diff --git a/src/1-one/tokenize/methods/03-whitespace/tokenize.js b/src/1-one/tokenize/methods/03-whitespace/tokenize.js
new file mode 100644
index 0000000..576cf10
--- /dev/null
+++ b/src/1-one/tokenize/methods/03-whitespace/tokenize.js
@@ -0,0 +1,88 @@
+//all punctuation marks, from https://en.wikipedia.org/wiki/Punctuation
+
+//we have slightly different rules for start/end - like #hashtags.
+const isLetter = /\p{Letter}/u
+const isNumber = /[\p{Number}\p{Currency_Symbol}]/u
+const hasAcronym = /^[a-z]\.([a-z]\.)+/i
+const chillin = /[sn]['’]$/
+const isFullNumber = /^[(+\-]?\d+(th|st|nd|rd)?[)+\-]?$/
+
+const normalizePunctuation = function (str, model) {
+  // quick lookup for allowed pre/post punctuation
+  const { prePunctuation, postPunctuation, emoticons } = model.one
+  const original = str
+  let pre = ''
+  let post = ''
+  const chars = Array.from(str)
+  let start = 0
+  let end = chars.length
+
+  // punctuation-only words, like '<3'
+  if (Object.hasOwn(emoticons, str.trim())) {
+    return { str: str.trim(), pre, post: ' ' } //not great
+  }
+
+  // Locate the first retained code point without shifting the array.
+  while (start < end) {
+    const c = chars[start]
+    // keep any declared chars
+    if (prePunctuation[c] === true) {
+      break//keep it
+    }
+    // keep '+' or '-' only before a number
+    if ((c === '+' || c === '-' || c === '(') && isFullNumber.test(str.trim())) {
+      break//done
+    }
+    // '97 - year short-form
+    if (c === "'" && c.length === 3 && isNumber.test(chars[start + 1])) {
+      break//done
+    }
+    // start of word
+    if (isLetter.test(c) || isNumber.test(c)) {
+      break //done
+    }
+    // punctuation
+    start++
+  }
+
+  // Locate the last retained code point.
+  while (end > start) {
+    const c = chars[end - 1]
+    // keep any declared chars
+    if (postPunctuation[c] === true) {
+      break//keep it
+    }
+    // start of word
+    if (isLetter.test(c) || isNumber.test(c)) {
+      break //done
+    }
+    // F.B.I.
+    if (c === '.' && hasAcronym.test(original) === true) {
+      break//keep it
+    }
+    //  keep s-apostrophe - "flanders'" or "chillin'"
+    if (c === "'" && chillin.test(original) === true) {
+      break//keep it
+    }
+    // keep '+' or ')' only for a number like (800) or 500+
+    if ((c === '+' || c === ')') && isFullNumber.test(str.trim())) {
+      break//done
+    }
+    // punctuation
+    end--
+  }
+  pre = chars.slice(0, start).join('')
+  post = chars.slice(end).join('')
+  str = chars.slice(start, end).join('')
+  //we went too far..
+  if (str === '') {
+    // do a very mild parse, and hope for the best.
+    let last = original.length
+    while (last > 0 && original[last - 1] === ' ') last--
+    post = original.slice(last)
+    str = original.slice(0, last)
+    pre = ''
+  }
+  return { str, pre, post }
+}
+export default normalizePunctuation
diff --git a/tests/one/tokenize/punctuation.test.js b/tests/one/tokenize/punctuation.test.js
new file mode 100644
index 0000000..eaab3e5
--- /dev/null
+++ b/tests/one/tokenize/punctuation.test.js
@@ -0,0 +1,54 @@
+import test from 'tape'
+import nlp from '../../two/_lib.js'
+const here = '[one/term-punctuation] '
+
+
+test('term punctuation', function (t) {
+  const arr = [
+    [`yeah??`, 'yeah'],
+    [`#canada`, 'canada'],
+    [`@canada`, 'canada'],
+    [`the  "gouvernement" party`, 'gouvernement'],
+    [`i guess... but`, 'guess'],
+    [`he did. (but barely)`, 'but barely'],
+    [`~word~`, 'word'],
+    [`'word'`, 'word'],
+    [`(word)`, 'word'],
+    [`([word])`, 'word'],
+    [`{word}`, 'word'],
+    [`-word-`, 'word'],
+    [`«‛“word〉`, 'word'],
+    [`'word'`, 'word'],
+    [`flanders'`, `flanders'`],
+    // [`_word_`, 'word'],
+  ]
+  arr.forEach(a => {
+    const [txt, match] = a
+    t.equal(nlp(txt).has(match), true, here + `"${txt}"`)
+  })
+  t.end()
+})
+
+test('modify existing punctuation', function (t) {
+  const world = nlp.world()
+
+  let term = nlp('=cool=').docs[0][0]
+  t.equal(term.normal, 'cool', here + 'before')
+
+  // change it
+  world.model.one.prePunctuation['='] = true
+  term = nlp('=cool=').docs[0][0]
+  t.equal(term.normal, '=cool', here + 'allow before')
+
+  world.model.one.postPunctuation['='] = true
+  term = nlp('=cool=').docs[0][0]
+  t.equal(term.normal, '=cool=', here + 'both')
+
+  world.model.one.prePunctuation['='] = false
+  world.model.one.postPunctuation['='] = false
+  term = nlp('=cool=').docs[0][0]
+  t.equal(term.normal, 'cool', here + 'fixed')
+
+
+  t.end()
+})
\ No newline at end of file