Describe this repository

README.md

@@ -0,0 +1,30 @@
+# Contributions to spencermountain/compromise
+
+modest natural-language processing
+
+This repository holds only my commits to the upstream project, with their original author, date and diff.
+Each of my commits is preceded by a snapshot of the upstream files it touched, so every diff shows exactly what I changed.
+
+## Merged pull requests
+
+- [find payloads by term id so they survive a .remove()](https://github.com/spencermountain/compromise/pull/1246) (2026-10-06)
+- [keep a titlecase comparative before 'than' as an adjective](https://github.com/spencermountain/compromise/pull/1242) (2026-10-06)
+- [keep a closing bracket in the word when it closes one inside it](https://github.com/spencermountain/compromise/pull/1249) (2026-10-05)
+- [match lexicon words that start with # or @](https://github.com/spencermountain/compromise/pull/1248) (2026-10-05)
+- [leave a spaced dash out of a match's text](https://github.com/spencermountain/compromise/pull/1247) (2026-10-05)
+- [contract each match on its own](https://github.com/spencermountain/compromise/pull/1244) (2026-10-05)
+- [don't add an implicit 'to' after 'wanna' before a noun](https://github.com/spencermountain/compromise/pull/1243) (2026-10-05)
+- [split hyphens in a match the same way the tokenizer does](https://github.com/spencermountain/compromise/pull/1241) (2026-10-05)
+- [contract 'he is not' to he's not instead of he'sn't](https://github.com/spencermountain/compromise/pull/1240) (2026-10-05)
+- [drop does/did when turning a negative verb positive](https://github.com/spencermountain/compromise/pull/1239) (2026-10-05)
+- [contract 'can not' to can't](https://github.com/spencermountain/compromise/pull/1237) (2026-10-05)
+- [dates: read singular possessive durations](https://github.com/spencermountain/compromise/pull/1236) (2026-10-05)
+- [don't merge an -ent syllable back into an open first syllable](https://github.com/spencermountain/compromise/pull/1233) (2026-10-05)
+- [run the regex-boundaries tests on the full build](https://github.com/spencermountain/compromise/pull/1232) (2026-10-05)
+- [parse 'fortnight' as two weeks](https://github.com/spencermountain/compromise/pull/1231) (2026-10-05)
+- [Treat 's before a copula as possessive](https://github.com/spencermountain/compromise/pull/1230) (2026-09-29)
+- [Keep the case in toSingular](https://github.com/spencermountain/compromise/pull/1229) (2026-09-29)
+- [Keep "never" when going to the infinitive](https://github.com/spencermountain/compromise/pull/1228) (2026-09-29)
+- [Skip the verb of a relative clause following the subject](https://github.com/spencermountain/compromise/pull/1225) (2026-09-19)
+- [Recognize more openers of a secondary clause in mainClause](https://github.com/spencermountain/compromise/pull/1224) (2026-09-19)
+- [Only treat subordinators as secondary-clause signals before the verb](https://github.com/spencermountain/compromise/pull/1222) (2026-09-14)

src/1-one/change/api/replace.js

@@ -0,0 +1,155 @@
+import fmts from '../../output/api/_fmts.js'
+
+const dollarStub = /\$[0-9a-z]+/g
+const fns = {}
+
+// case logic
+const isTitleCase = (str) => /^\p{Lu}[\p{Ll}'’]/u.test(str) || /^\p{Lu}$/u.test(str)
+const toTitleCase = (str) => str.replace(/^\p{Ll}/u, x => x.toUpperCase())
+const toLowerCase = (str) => str.replace(/^\p{Lu}/u, x => x.toLowerCase())
+
+// the match's pre and post are kept by replaceWith(), so m.text() should leave them out
+const withoutEnds = function (m) {
+  const text = m.text
+  m.text = fmt => {
+    const opts = typeof fmt === 'string' ? fmts[fmt] : fmt
+    return text.call(m, { keepEndPunct: false, ...opts })
+  }
+  return m
+}
+
+// put the match's own punctuation around the replacement's, unless it already has it
+const outerPre = (pre, fresh) => {
+  const punct = pre.trimStart()
+  return fresh.startsWith(punct) ? pre.slice(0, pre.length - punct.length) + fresh : pre + fresh
+}
+const outerPost = (post, fresh) => {
+  const punct = post.trimEnd()
+  return fresh.endsWith(punct) ? fresh + post.slice(punct.length) : fresh + post
+}
+
+// doc.replace('foo', (m)=>{})
+const replaceByFn = function (main, fn, keep) {
+  main.forEach(m => {
+    const out = fn(withoutEnds(m))
+    m.replaceWith(out, keep)
+  })
+  return main
+}
+
+// support 'foo $0' replacements
+const subDollarSign = function (input, main) {
+  if (typeof input !== 'string') {
+    return input
+  }
+  const groups = main.groups()
+  input = input.replace(dollarStub, a => {
+    const num = a.replace(/\$/, '')
+    if (Object.hasOwn(groups, num)) {
+      return groups[num].text()
+    }
+    return a
+  })
+  return input
+}
+
+fns.replaceWith = function (input, keep = {}) {
+  let ptrs = this.fullPointer
+  // support keep-all option
+  if (keep === true) {
+    keep = {
+      tags: true,
+      case: true,
+      possessives: true,
+    }
+  }
+  const main = this
+  this.uncache()
+  if (typeof input === 'function') {
+    return replaceByFn(main, input, keep)
+  }
+  const terms = main.docs[0]
+  if (!terms) return main
+  const isOriginalPossessive = keep.possessives && terms.at(-1).tags.has('Possessive')
+  const isOriginalTitleCase = keep.case && isTitleCase(terms[0].text)
+  // support 'foo $0' replacements
+  input = subDollarSign(input, main)
+
+  const original = this.update(ptrs)
+  // soften-up pointer
+  ptrs = ptrs.map(ptr => ptr.slice(0, 3))
+  // original.freeze()
+  let oldTags = (original.docs[0] || []).map(term => Array.from(term.tags))
+  const originalEnds = original.docs.map(words => [words[0].pre, words.at(-1).post])
+  // slide this in
+  if (typeof input === 'string') {
+    input = this.fromText(input).compute('id')
+  }
+  const fresh = input.docs[0]
+  main.insertAfter(input)
+  // keep the punctuation around each match, outside the replacement's own
+  if (fresh) {
+    main.docs.forEach((words, i) => {
+      const [pre, post] = originalEnds[i]
+      words.at(-fresh.length).pre = outerPre(pre, fresh[0].pre.trimStart())
+      if (post) {
+        words.at(-1).post = outerPost(post, fresh.at(-1).post.trimEnd())
+      }
+    })
+  }
+  // are we replacing part of a contraction?
+  if (original.has('@hasContraction') && main.contractions) {
+    const more = main.grow('@hasContraction+')
+    more.contractions().expand()
+  }
+  // delete the original terms
+  main.delete(original) //science.
+
+  // keep "John's"
+  if (isOriginalPossessive) {
+    const tmp = main.docs[0]
+    const term = tmp.at(-1)
+    if (!term.tags.has('Possessive')) {
+      term.text += "'s"
+      term.normal += "'s"
+      term.tags.add('Possessive')
+    }
+  }
+
+  // what should we return?
+  const m = main.toView(ptrs).compute(['index', 'freeze', 'lexicon'])
+  if (m.world.compute.preTagger) {
+    m.compute('preTagger')
+  }
+  m.compute('unfreeze')
+  // replace any old tags
+  if (keep.tags) {
+    // truncate old tags to only touch new terms
+    oldTags = oldTags.slice(0, input.wordCount())
+    m.terms().forEach((term, i) => {
+      term.tagSafe(oldTags[i])
+    })
+  }
+
+  if (!m.docs[0] || !m.docs[0][0]) return m
+
+  // try to co-erce case, too
+  if (keep.case) {
+    const transformCase = isOriginalTitleCase ? toTitleCase : toLowerCase
+    m.docs[0][0].text = transformCase(m.docs[0][0].text)
+  }
+  return m
+}
+
+fns.replace = function (match, input, keep) {
+  if (match && !input) {
+    return this.replaceWith(match, keep)
+  }
+  const m = this.match(match)
+  if (!m.found) {
+    return this
+  }
+  this.soften()
+  return m.replaceWith(input, keep)
+}
+export default fns

src/1-one/output/api/html.js

@@ -60,11 +60,11 @@ const html = function (obj) {
   this.docs.forEach(terms => {
     for (let i = 0; i < terms.length; i += 1) {
       const t = terms[i]
-      // do a span tag
+      out += t.pre || ''
+      // do a span tag, inside the pre-punctuation
       if (Object.hasOwn(starts, t.id)) {
         out += starts[t.id].join('')
       }
-      out += t.pre || ''
       out += t.text || ''
       if (Object.hasOwn(ends, t.id)) {
         out += ends[t.id].join('')

src/1-one/output/api/wrap.js

@@ -0,0 +1,52 @@
+import fmts from './_fmts.js'
+
+const toText = function (term) {
+  const pre = term.pre || ''
+  const post = term.post || ''
+  return pre + term.text + post
+}
+
+// the match's pre and post are added around fn(m), so m.text() should leave them out
+const withoutEnds = function (m) {
+  const text = m.text
+  m.text = fmt => {
+    const opts = typeof fmt === 'string' ? fmts[fmt] : fmt
+    return text.call(m, { keepEndPunct: false, ...opts })
+  }
+  return m
+}
+
+const findStarts = function (doc, obj) {
+  const starts = {}
+  Object.keys(obj).forEach(reg => {
+    const m = doc.match(reg)
+    m.fullPointer.forEach(a => {
+      starts[a[3]] = { fn: obj[reg], end: a[2] }
+    })
+  })
+  return starts
+}
+
+const wrap = function (doc, obj) {
+  // index ids to highlight
+  const starts = findStarts(doc, obj)
+  let text = ''
+  doc.docs.forEach((terms, n) => {
+    for (let i = 0; i < terms.length; i += 1) {
+      const t = terms[i]
+      // do a span tag
+      if (Object.hasOwn(starts, t.id)) {
+        const { fn, end } = starts[t.id]
+        const m = withoutEnds(doc.update([[n, i, end]]))
+        text += terms[i].pre || ''
+        text += fn(m)
+        i = end - 1
+        text += terms[i].post || ''
+      } else {
+        text += toText(t)
+      }
+    }
+  })
+  return text
+}
+export default wrap

src/1-one/tokenize/methods/03-whitespace/tokenize.js

@@ -2,6 +2,9 @@
 
 //we have slightly different rules for start/end - like #hashtags.
 const isLetter = /\p{Letter}/u
+const hasMark = /\p{Mark}/u
+// a letter with its combining marks, or any one char
+const splitChars = /\p{Letter}\p{Mark}*|./gsu
 const isNumber = /[\p{Number}\p{Currency_Symbol}]/u
 const hasAcronym = /^[a-z]\.([a-z]\.)+/i
 const chillin = /[sn]['’]$/
@@ -28,7 +31,8 @@ const normalizePunctuation = function (str, model) {
   const original = str
   let pre = ''
   let post = ''
-  const chars = Array.from(str)
+  // keep combining marks on their letter - 'cafe\u0301'
+  const chars = hasMark.test(str) ? str.match(splitChars) : Array.from(str)
   let start = 0
   let end = chars.length
 

tests/01-one/output/html.test.js

@@ -59,3 +59,19 @@ test('html-implicit', function (t) {
   t.equal(out, `he's cool`, here + 'implict')
   t.end()
 })
+
+test('html-punctuation', function (t) {
+  let doc = nlp(`This is a "test". Hello "World."`)
+  let html = doc.html({ '.red': '(test|world)' })
+  t.equal(html, `This is a "<span class="red">test</span>". Hello "<span class="red">World</span>."`, here + 'quotes')
+
+  doc = nlp(`he said "hello world" twice`)
+  html = doc.html({ '.red': 'hello world' })
+  t.equal(html, `he said "<span class="red">hello world</span>" twice`, here + 'quoted phrase')
+
+  doc = nlp(`my dog (spot) is cute`)
+  html = doc.html({ '.red': 'spot' })
+  t.equal(html, `my dog (<span class="red">spot</span>) is cute`, here + 'parentheses')
+
+  t.end()
+})

tests/01-one/tokenize/unicode.test.js

@@ -0,0 +1,66 @@
+import test from 'tape'
+import nlp from '../../lib/one.js'
+const here = '[one/tokenize/unicode] '
+
+test('many-unicode', function (t) {
+  const str = `✐✠✰❀❐❞❰➀➐➠➰✁✑✡✱❁❑❡❱➁➑➡➱✂✒✢✲❂❒❢❲➂➒➢➲✃✓✣✳❃❓❣❳➃➓➣➳✄✔✤✴❄❔❤❴➄➔➤➴✅✕✥✵❅❕❥❵➅➕➥➵✆✖✦✶❆❖❦❶➆➖➦➶✇✗✧✷❇❗❧❷➇➗➧➷✈✘✨✸❈❘❨❸➈➘➨➸✉✙✩✹❉❙❩❹➉➙➩➹✊✚✪✺❊❚❪❺➊➚➪➺✋✛✫✻❋❛❫❻➋➛➫➻✌✜✬✼❌❜❬❼➌➜➬➼✍✝✭✽❍❝❭❽➍➝➭➽✎✞✮✾❎❞❮❾➎➞➮➾✏✟✯✿❏❜❯❿➏➟➯➿😀😐😠😰🙀😁😑😡😱🙁😂😒😢😲🙂😃😓😣😳🙃😄😔😤😴🙄😅😕😥😵🙅😆😖😦😶🙆😇😗😧😷🙇😈😘😨😸🙈😉😙😩😹🙉😊😚😪😺🙊😋😛😫😻🙋😌😜😬😼🙌😍😝😭😽🙍😎😞😮😾🙎😏😟😯😿🙏,&、*.+-;<:>?=!—\($)%{@}〔〕₠₰₡₱₢₲₣₳₤₴₥₵₦₶₧₷₸₩₹₪₺₫₻€₼₭₽₮₾₯₿` // eslint-dis
+  const doc = nlp(str)
+  t.equal(doc.text(), str, here + 'identical-text')
+  t.equal(doc.length, 1, here + 'one-sentence')
+  t.equal(doc.terms().length, 1, here + 'one-term')
+  t.end()
+})
+
+test('em-dashes', function (t) {
+  const str = 'text—text'
+  const doc = nlp(str)
+  t.equal(doc.text() === str, true, here + 'emdash')
+  t.end()
+})
+
+// this section is very cursed
+test('zero-width-chars', function (t) {
+  //this has a zero-width character
+  // eslint-disable-next-line no-irregular-whitespace -- Intentional tokenization fixture.
+  const str = `before​ after`
+  const doc = nlp(str)
+  t.equal(doc.text(), str, here + 'zero-width passes-through')
+  const json = doc.json({ terms: { normal: true } })
+  const before = json[0].terms[0]
+  t.equal(before.normal, 'before', here + 'normalized-out in json')
+  t.equal(before.post, ' ', here + 'normal whitespace in json')
+  t.ok(doc.text() !== 'before after', here + 'default text has 0-width-char')
+  t.equal(doc.text('normal'), 'before after', here + 'normal text removes 0-width-char')
+  t.equal(doc.text('clean'), 'before after', here + 'clean text removes 0-width-char')
+  t.equal(doc.text('reduced'), 'before after', here + 'reduced text removes 0-width-char')
+  t.equal(doc.text('root'), 'before after', here + 'root text removes 0-width-char')
+  t.end()
+})
+
+test('fullwidth-ascii', function (t) {
+  t.equal(nlp('ＡＢＣ').has('abc'), true, here + 'fullwidth match')
+  t.equal(nlp('Ｈｅｌｌｏ ２０２４').out('normal'), 'hello 2024', here + 'fullwidth normal')
+  t.equal(nlp('Ｈｅｌｌｏ ２０２４').text(), 'Ｈｅｌｌｏ ２０２４', here + 'fullwidth text unchanged')
+  t.end()
+})
+
+test('combining-marks', function (t) {
+  const arr = [
+    ['caf\u00e9 au lait'.normalize('NFD'), 'cafe\u0301', ' '],
+    ['Poincare\u0301', 'Poincare\u0301', ''],
+    ['E\u0300\u0304 x', 'E\u0300\u0304', ' '],
+    ['(cafe\u0301)', 'cafe\u0301', ')'],
+    ['cafe\u0301!', 'cafe\u0301', '!'],
+    ['\u0928\u092e\u0938\u094d\u0924\u0947 x', '\u0928\u092e\u0938\u094d\u0924\u0947', ' '],
+    ['\u2764\ufe0f x', '\u2764\ufe0f', ' '],
+  ]
+  arr.forEach(a => {
+    const [str, text, post] = a
+    const doc = nlp(str)
+    const term = doc.json()[0].terms[0]
+    t.equal(term.text, text, here + `text "${str}"`)
+    t.equal(term.post, post, here + `post "${str}"`)
+    t.equal(doc.text(), str, here + `identical-text "${str}"`)
+  })
+  t.end()
+})

tests/02-two/change/replace.test.js

@@ -0,0 +1,267 @@
+import test from 'tape'
+import nlp from '../../lib/two.js'
+const here = '[two/change/replace] '
+
+test('replace-basic :', function (t) {
+  let m = nlp('the dog played').match('dog').replace('cat').all()
+  t.equal(m.out('text'), 'the cat played', here + 'dog-cat')
+
+  m = nlp('the dog played').match('the dog').replace('a cat').all()
+  t.equal(m.out('text'), 'a cat played', here + 'a-cat')
+
+  m = nlp('the dog played').match('#Noun').replace('snake').all()
+  t.equal(m.out('text'), 'the snake played', here + 'snake')
+
+  m = nlp('the pit bull played').match('#Noun+').replace('snake').all()
+  t.equal(m.out('text'), 'the snake played', here + 'pit bull')
+
+  m = nlp('the pit bull dog played').match('#Noun+').replace('grey snake').all()
+  t.equal(m.out('text'), 'the grey snake played', here + 'pit bull dog')
+
+  t.end()
+})
+
+test('match-replace :', function (t) {
+  const arr = [
+    ['the dog played', 'the dog', 'the cat', 'the cat played'],
+    ['the dog played', 'the #Noun', 'the cat', 'the cat played'],
+    ['the dog played', 'the (dog|hamster|pet-snake)', 'the cat', 'the cat played'],
+    ['the boy and the girl', 'the #Noun', 'the house', 'the house and the house'],
+    ['the boy and the girl', 'the cat', 'the house', 'the boy and the girl'],
+    ['the boy and the girl', '(boy|girl)', 'cat', 'the cat and the cat'],
+  ]
+  arr.forEach(function (a) {
+    const str = nlp(a[0]).replace(a[1], a[2]).all().out('text')
+    const msg = str + ' -- ' + a[3]
+    t.equal(str, a[3], here + msg)
+  })
+  t.end()
+})
+
+test('replace-with-punctuation', function (t) {
+  const doc = nlp('Simon, how is Pamela and Jason?')
+  const str = doc.match('#Person').replace('PERSON').all().out()
+  t.equal(str, 'PERSON, how is PERSON and PERSON?', here + 'replace-with-punctuation')
+  t.end()
+})
+
+test('structured-object-replace :', function (t) {
+  let r = nlp('fun times in cool town')
+  const term = r.match('times')
+  r.replace(term, 'day')
+  t.equal(r.out(), 'fun day in cool town', here + 'structured-replace')
+
+  r = nlp('fun times in cool town')
+  const terms = r.match('cool town')
+  r.replace(terms, 'shitsville')
+  t.equal(r.out(), 'fun times in shitsville', here + 'structured-replace-multi')
+  t.end()
+})
+
+test('replace-keep some punctuation', function (t) {
+  let doc = nlp('two weeks').tag('Cool')
+  doc.replace('two', '2', { tags: true })
+  t.equal(doc.match('#Cool+').text(), '2 weeks', here + 'replace-keep-tags')
+
+  doc = nlp('first sentence. I am trying it out.')
+  const m = doc.match('trying').tag('HashTag')
+  m.replaceWith('working', { tags: true })
+  t.equal(doc.match('#HashTag+').text(), 'working', here + 'replacewith-keep-tags')
+  t.end()
+})
+
+test('replace over implict', function (t) {
+  const doc = nlp("i'm good")
+  doc.match('am').replaceWith('was')
+  t.equal(doc.text(), 'i was good', here + 'replace over implicit')
+  t.end()
+})
+
+test('replace-with-Doc', function (t) {
+  const b = nlp('sneaks').tag('Cool')
+  const doc = nlp(`john walks quickly`)
+  doc.match('walks').replaceWith(b, { tags: true })
+  t.equal(doc.text(), 'john sneaks quickly', here + 'doc-replace')
+  t.equal(doc.has('#Cool'), true, here + 'doc-replace tags')
+  t.end()
+})
+
+test('replace-with-function', function (t) {
+  const fn = p => {
+    if (p.has('john')) {
+      return 'johnny'
+    }
+    return 'nancy'
+  }
+  let doc = nlp('spencer and John')
+  doc.replace('#Person', fn)
+  t.equal(doc.text(), 'nancy and johnny', here + 'replace function')
+
+  doc = nlp('spencer and John')
+  doc.match('#Person').replaceWith(fn, { case: true })
+  t.equal(doc.text(), 'nancy and Johnny')
+
+  doc = nlp('Thurs, Feb 2nd, 2016')
+  doc.match('feb').replaceWith(m => {
+    return m.text({ trim: true }) + '!'
+  })
+  t.equal(doc.text(), 'Thurs, Feb! 2nd, 2016', here + 'replaceWith function')
+
+  doc = nlp('Spencer is very cool.')
+  doc.match('spencer').replaceWith((m) => m.text() + 's')
+  doc.match('is').replaceWith(() => 'are')
+  t.equal(doc.text(), 'Spencers are very cool.', here + 'replaceWith twice')
+  t.end()
+})
+
+test('replace-with-possessives', function (t) {
+  const fn = p => {
+    if (p.has('john')) {
+      return 'johnny'
+    }
+    return 'nancy'
+  }
+  const doc = nlp('spencer\'s house is cooler than John\'s house.')
+  doc.replace('#Person', fn, { case: true, possessives: true })
+  t.equal(doc.text(), 'nancy\'s house is cooler than Johnny\'s house.')
+  t.end()
+})
+
+test('replace-tags-param', function (t) {
+  let doc = nlp('Spencer is very cool.')
+  doc.match('spencer').replaceWith('jogging')
+  const m = doc.match('jogging')
+  t.equal(m.has('#Person'), false, here + 'tags not-kept - default')
+
+  doc = nlp('Spencer is very cool.')
+  doc.match('spencer').replaceWith('jogging', { tags: true })
+  t.equal(doc.has('(jogging && #Person)'), true, here + 'tags kept - boolean')
+  t.equal(doc.has('(jogging && #Gerund)'), false, here + 'tags kept - boolean')
+
+  doc = nlp('Spencer is very cool.')
+  doc.match('spencer').replaceWith('jogging', { tags: true })
+  t.equal(doc.has('(jogging && #Person)'), true, here + 'tags kept - boolean')
+  t.equal(doc.has('(jogging && #Gerund)'), false, 'here+tags kept - boolean')
+
+  t.end()
+})
+
+test('replace-case-param', function (t) {
+  let doc = nlp('Spencer is very cool.')
+  doc.match('spencer').replaceWith('jogging', { case: true })
+  t.equal(doc.text(), 'Jogging is very cool.', here + here + 'case kept ')
+
+  doc = nlp('spencer is very cool.')
+  doc.match('spencer').replaceWith('jogging', { case: false }) // Jogging?
+  t.equal(doc.text(), 'jogging is very cool.', here + here + 'lowsercase kept - default')
+
+  doc = nlp('Spencer is very cool.')
+  doc.match('spencer').replaceWith('jogging', { case: false })
+  t.equal(doc.text(), 'jogging is very cool.', 'dont-keep')
+  t.end()
+})
+
+test('replace-repair', function (t) {
+  let doc = nlp('the cat and the dog')
+  doc.replace('#Noun', 'house')
+  t.equal(doc.text(), 'the house and the house', here + 'repair-one-one')
+
+
+  doc = nlp('the cat and the dog')
+  doc.replace('the #Noun', 'the house')
+  t.equal(doc.text(), 'the house and the house', here + 'repair-two-two')
+
+  doc = nlp('the cat and the dog')
+  doc.replace('#Noun', 'cool house')
+  t.equal(doc.text(), 'the cool house and the cool house', here + 'repair-one-two')
+
+  doc = nlp('the cat and the dog')
+  doc.replace('the #Noun', 'house')
+  t.equal(doc.text(), 'house and house', here + 'repair-two-one')
+  t.end()
+})
+
+test('replace is cloned', function (t) {
+  const doc = nlp('before match after. second sentence here.')
+  const m = doc.match('match')
+  doc.match('sentence').replace(m)
+  t.equal(doc.text(), 'before match after. second match here.', here + 'replaced-text')
+
+  const id = m.docs[0][0].id
+  const foundIds = doc.termList().filter(term => term.id === id)
+  t.equal(foundIds.length, 1, 'id-different')
+
+  t.equal(m.length, 1, 'match-unchanged')
+  m.tag('NewTag')
+  t.equal(m.match('#NewTag').length, 1, 'tags-are-cloned')
+
+  t.end()
+})
+
+test('replace-with-no-match', function (t) {
+  const doc = nlp('original')
+  doc.match('missing').replaceWith('unreached')
+  t.equal(doc.text(), 'original')
+
+  doc.match('missing').replaceWith('unreached', { tags: true, possessives: true, case: true })
+  t.equal(doc.text(), 'original')
+  t.end()
+})
+
+test('replace-with-pre-post', function (t) {
+  let doc = nlp('`Target`')
+  doc.match('Target').replaceWith('Replacement')
+  t.equal(doc.text(), '`Replacement`')
+
+  doc = nlp('`John\'s`')
+  doc.match('John').replaceWith('Spencer', { possessives: true })
+  t.equal(doc.text(), '`Spencer\'s`')
+
+  doc = nlp('the `pit bull` played').match('#Noun+').replace('snake').all()
+  t.equal(doc.text(), 'the `snake` played')
+
+  doc = nlp('the `snake` played').match('#Noun+').replace('pit bull').all()
+  t.equal(doc.text(), 'the `pit bull` played')
+
+  t.end()
+})
+
+test('replace-edge-punctuation', function (t) {
+  let arr = [
+    ['"hello world" he said', '"[x]" he said'],
+    ['he said "hello world" today', 'he said "[x]" today'],
+    ['he said "hello world"', 'he said "[x]"'],
+    ['(hello world) he said', '([x]) he said'],
+    ['he said (hello world) today', 'he said ([x]) today'],
+    ['he said hello world.', 'he said [x].'],
+    ['"hello world" and "hello world" too', '"[x]" and "[x]" too'],
+  ]
+  arr.forEach(a => {
+    const doc = nlp(a[0])
+    doc.replace('hello world', '[x]')
+    t.equal(doc.text(), a[1], here + 'string: ' + a[0])
+  })
+
+  arr = [
+    ['"hello world" he said', '"HELLO WORLD" he said'],
+    ['he said "hello world" today', 'he said "HELLO WORLD" today'],
+    ['he said "hello world"', 'he said "HELLO WORLD"'],
+    ['(hello world) he said', '(HELLO WORLD) he said'],
+    ['hello world.', 'HELLO WORLD.'],
+    ['"hello world" and "hello world" too', '"HELLO WORLD" and "HELLO WORLD" too'],
+  ]
+  arr.forEach(a => {
+    const doc = nlp(a[0])
+    doc.replace('hello world', m => m.text().toUpperCase())
+    t.equal(doc.text(), a[1], here + 'function: ' + a[0])
+  })
+
+  let doc = nlp('"hello world" he said')
+  doc.replace('hello world', m => '[' + m.text() + ']')
+  t.equal(doc.text(), '"[hello world]" he said', here + 'function with brackets')
+
+  doc = nlp('he said "hello" today')
+  doc.replace('hello', '"hi"')
+  t.equal(doc.text(), 'he said "hi" today', here + 'quotes already in replacement')
+  t.end()
+})

tests/02-two/output/out.test.js

@@ -0,0 +1,66 @@
+import test from 'tape'
+import nlp from '../../lib/two.js'
+const here = '[two/output/out] '
+
+test('out-tags', function (t) {
+  const out = nlp(`he's cool.`).out('tags')
+  t.equal(out.length, 1, 'one sentence')
+  t.equal(out[0].he.indexOf('Pronoun') !== -1, true, here + 'has he:Pronoun')
+  t.equal(out[0].is.indexOf('Copula') !== -1, true, here + 'has is:Copula')
+  t.equal(out[0].cool.indexOf('Adjective') !== -1, true, here + 'has cool:Adjective')
+  t.end()
+})
+
+test('out-topk', function (t) {
+  const doc = nlp(`What'd I say? Monorail. and it called? Monorail.`)
+  const out = doc.out('freq')
+  // t.equal(out.length, 3, here + 'three results')
+  t.equal(out[0].normal, 'monorail.', here + 'top-result is most-frequent')
+  t.equal(out[0].count, 2, here + 'got count')
+  t.end()
+})
+
+test('out-array', function (t) {
+  const arr = nlp('1-2').terms().out('array')
+  t.equal(arr.length, 1, here + 'one result')
+  t.equal(arr[0], '1-2', here + 'got contraction text')
+  t.end()
+})
+
+test('out-wrap', function (t) {
+  let doc = nlp("soft and yielding like a nerf ball")
+  let out = doc.out({
+    '#Adjective': (m) => `[${m.text()}]`
+  })
+  t.equal(out, `[soft] and [yielding] like a nerf ball`, here + 'two matches')
+
+  // pre-post
+  doc = nlp("before (match) after")
+  out = doc.wrap({
+    'match': () => `few more words`,
+  })
+  t.equal(out, `before (few more words) after`, here + 'pre+post')
+  t.end()
+})
+
+test('out-wrap-punctuation', function (t) {
+  const arr = [
+    [`"hello world" he said`, `"[hello world]" he said`],
+    [`he said "hello world" today`, `he said "[hello world]" today`],
+    [`he said "hello world"`, `he said "[hello world]"`],
+    [`(hello world) he said`, `([hello world]) he said`],
+    [`he said (hello world) today`, `he said ([hello world]) today`],
+    [`he said (hello world).`, `he said ([hello world]).`],
+    [`hello world, he said`, `[hello world], he said`],
+    [`he left. "hello world" she said`, `he left. "[hello world]" she said`],
+    [`"hello world."`, `"[hello world]."`],
+  ]
+  arr.forEach(function (a) {
+    const doc = nlp(a[0])
+    t.equal(doc.out({ 'hello world': m => `[${m.text()}]` }), a[1], here + 'text: ' + a[0])
+    t.equal(doc.wrap({ 'hello world': m => `[${m.out('normal')}]` }), a[1], here + 'normal: ' + a[0])
+  })
+  const out = nlp(`"Hello World" he said`).wrap({ 'hello world': m => `[${m.text('normal')}]` })
+  t.equal(out, `"[hello world]" he said`, here + 'keeps text format')
+  t.end()
+})

tests/one/output/html.test.js

@@ -0,0 +1,77 @@
+import test from 'tape'
+import nlp from '../_lib.js'
+const here = '[one/html] '
+
+test('html-match', function (t) {
+  let doc = nlp(`match one two. one match two. one two match.`)
+  let html = doc.html({ i: 'match' })
+  t.equal(html, `<i>match</i> one two. one <i>match</i> two. one two <i>match</i>.`, here + 'html tag')
+
+  doc = nlp(`match one two. one match two.`)
+  html = doc.html({ '.foo': 'match' })
+  t.equal(html, `<span class="foo">match</span> one two. one <span class="foo">match</span> two.`, here + 'html class')
+
+  doc = nlp(`one match match two.`)
+  html = doc.html({ '#foo': 'match+' })
+  t.equal(html, `one <span id="foo">match match</span> two.`, here + 'html id')
+
+  doc = nlp(`one match two.`)
+  html = doc.html({ '.red': 'match+', '.blue': doc.match('two') })
+  t.equal(html, `one <span class="red">match</span> <span class="blue">two</span>.`, here + 'html two classes')
+
+  doc = nlp(`if i can recall, my grey dog loves pizza crusts (they are really good).`)
+  html = doc.html({ '.red': 'my grey dog', '.blue': doc.match('loves') })
+  t.equal(
+    html,
+    `if i can recall, <span class="red">my grey dog</span> <span class="blue">loves</span> pizza crusts (they are really good).`,
+    here + 'html pre test'
+  )
+
+  t.end()
+})
+
+test('html-nest', function (t) {
+  const doc = nlp(`one match two.`)
+  let html = doc.html({
+    i: 'match',
+    b: 'one match two',
+  })
+  t.equal(html, `<b>one <i>match</i> two</b>.`, here + 'easy nest')
+
+  html = doc.html({
+    b: 'match two',
+    i: 'match',
+  })
+  t.equal(html, `one <b><i>match</i> two</b>.`, here + 'hard nest')
+
+  // html = doc.html({
+  //   i: 'match',
+  //   b: 'match two',
+  // })
+  // t.equal(html, `one <b><i>match</i> two</b>.`, here + 'harder nest')
+
+  t.end()
+})
+
+test('html-implicit', function (t) {
+  const doc = nlp(`he's cool`)
+  const out = doc.html({ '.foo': '#Verb' })
+  t.equal(out, `he's cool`, here + 'implict')
+  t.end()
+})
+
+test('html-punctuation', function (t) {
+  let doc = nlp(`This is a "test". Hello "World."`)
+  let html = doc.html({ '.red': '(test|world)' })
+  t.equal(html, `This is a "<span class="red">test</span>". Hello "<span class="red">World</span>."`, here + 'quotes')
+
+  doc = nlp(`he said "hello world" twice`)
+  html = doc.html({ '.red': 'hello world' })
+  t.equal(html, `he said "<span class="red">hello world</span>" twice`, here + 'quoted phrase')
+
+  doc = nlp(`my dog (spot) is cute`)
+  html = doc.html({ '.red': 'spot' })
+  t.equal(html, `my dog (<span class="red">spot</span>) is cute`, here + 'parentheses')
+
+  t.end()
+})