Upstream spencermountain/compromise state of the touched files before 16b6093898

plugins/dates/tests/regressions.test.js

@@ -167,11 +167,3 @@ test('a range does not stop other dates from splitting', (t) => {
   t.equal(res.length, 3, 'three dates found')
   t.end()
 })
-
-test('singular possessive durations', (t) => {
-  t.equal(getStart("in a month's time"), '2026-08-11T00:00:00.000-04:00', "a month's time")
-  t.equal(getStart("in a week's time"), '2026-07-18T00:00:00.000-04:00', "a week's time")
-  t.equal(getStart("in a year's time"), '2027-07-11T00:00:00.000-04:00', "a year's time")
-  t.equal(getStart("in two weeks' time"), '2026-07-25T00:00:00.000-04:00', "two weeks' time")
-  t.end()
-})

src/1-one/output/api/_text.js

@@ -0,0 +1,107 @@
+// Walk the trailing punctuation run once, rather than retrying at every offset.
+const endPunctuation = /[,:;)\]*.?~!\u0022\uFF02\u201D\u2019\u00BB\u203A\u2032\u2033\u2034\u301E\u00B4—-]/
+const trimEnd = str => {
+  let end = str.length
+  while (end > 0 && endPunctuation.test(str[end - 1])) end--
+  return str.slice(0, end)
+}
+const trimStart =
+  /^[(['"*~\uFF02\u201C\u2018\u201F\u201B\u201E\u2E42\u201A\u00AB\u2039\u2035\u2036\u2037\u301D\u0060\u301F]+/
+
+const punctToKill = /[,:;)('"\u201D\]]/
+const isHyphen = /^[-–—]$/
+const hasSpace = / /
+
+const textFromTerms = function (terms, opts, keepSpace = true) {
+  let txt = ''
+  terms.forEach(t => {
+    let pre = t.pre || ''
+    let post = t.post || ''
+    if (opts.punctuation === 'some') {
+      pre = pre.replace(trimStart, '')
+      // replace a hyphen with a space
+      if (isHyphen.test(post)) {
+        post = ' '
+      }
+      post = post.replace(punctToKill, '')
+      // cleanup exclamations
+      post = post.replace(/\?!+/, '?')
+      post = post.replace(/!+/, '!')
+      post = post.replace(/\?+/, '?')
+      // kill elipses
+      post = post.replace(/\.{2,}/, '')
+      // kill abbreviation periods
+      if (t.tags.has('Abbreviation')) {
+        post = post.replace(/\./, '')
+      }
+    }
+    if (opts.whitespace === 'some') {
+      pre = pre.replace(/\s/, '') //remove pre-whitespace
+      post = post.replace(/\s+/, ' ') //replace post-whitespace with a space
+    }
+    if (!opts.keepPunct) {
+      pre = pre.replace(trimStart, '')
+      if (post === '-') {
+        post = ' '
+      } else {
+        post = trimEnd(post)
+      }
+    }
+    // grab the correct word format
+    let word = t[opts.form || 'text'] || t.normal || ''
+    if (opts.form === 'implicit') {
+      word = t.implicit || t.text
+    }
+    if (opts.form === 'root' && t.implicit) {
+      word = t.root || t.implicit || t.normal
+    }
+    // add an implicit space, for contractions
+    if ((opts.form === 'machine' || opts.form === 'implicit' || opts.form === 'root') && t.implicit) {
+      if (!post || !hasSpace.test(post)) {
+        post += ' '
+      }
+    }
+    txt += pre + word + post
+  })
+  if (keepSpace === false) {
+    txt = txt.trim()
+  }
+  if (opts.lowerCase === true) {
+    txt = txt.toLowerCase()
+  }
+  return txt
+}
+
+const textFromDoc = function (docs, opts) {
+  let text = ''
+  if (!docs || !docs[0] || !docs[0][0]) {
+    return text
+  }
+  for (let i = 0; i < docs.length; i += 1) {
+    // middle
+    text += textFromTerms(docs[i], opts, true)
+  }
+  if (!opts.keepSpace) {
+    text = text.trim()
+  }
+  if (opts.keepEndPunct === false) {
+    // don't remove ':)' etc
+    if (!docs[0][0].tags.has('Emoticon')) {
+      text = text.replace(trimStart, '')
+    }
+    // remove ending periods
+    const last = docs.at(-1)
+    if (!last.at(-1).tags.has('Emoticon')) {
+      text = trimEnd(text)
+    }
+    // kill end quotations
+    if (text.endsWith(`'`) && !text.endsWith(`s'`)) {
+      text = text.replace(/'/, '')
+    }
+  }
+  if (opts.cleanWhitespace === true) {
+    text = text.trim()
+  }
+  return text
+}
+export { textFromDoc, textFromTerms }

src/1-one/output/api/text.js

@@ -0,0 +1,39 @@
+import { textFromDoc } from './_text.js'
+import fmts from './_fmts.js'
+
+const isObject = val => {
+  return Object.prototype.toString.call(val) === '[object Object]'
+}
+
+export default {
+  /** */
+  text: function (fmt) {
+    let opts = {}
+    if (fmt && typeof fmt === 'string' && Object.hasOwn(fmts, fmt)) {
+      opts = { ...fmts[fmt] }
+    } else if (fmt && isObject(fmt)) {
+      opts = { ...fmt } //todo: fixme
+    }
+    // is it a full document?
+    if (opts.keepSpace === undefined && !this.isFull()) {
+      //
+      opts.keepSpace = false
+    }
+    if (opts.keepEndPunct === undefined && this.pointer) {
+      const ptr = this.pointer[0]
+      if (ptr?.[1]) {
+        opts.keepEndPunct = false
+      } else {
+        opts.keepEndPunct = true
+      }
+    }
+    // set defaults
+    if (opts.keepPunct === undefined) {
+      opts.keepPunct = true
+    }
+    if (opts.keepSpace === undefined) {
+      opts.keepSpace = true
+    }
+    return textFromDoc(this.docs, opts)
+  },
+}

tests/one/change/whitespace.test.js

@@ -0,0 +1,76 @@
+import test from 'tape'
+import nlp from '../_lib.js'
+const here = '[one/whitespace] '
+
+test('whitespace-out', function (t) {
+  const str = 'one, two three. One, two, four?'
+  const doc = nlp(str)
+
+  t.equal(doc.out(), str, here + 'original-okay')
+
+  // some phrases, but full-phrases
+  t.equal(doc.eq(0).text(), 'one, two three.', here + '.eq(0) okay')
+  t.equal(doc.eq(1).text(), 'One, two, four?', here + '.eq(1) okay')
+
+  t.equal(doc.match('four').text(), 'four', here + 'one match')
+
+  t.equal(doc.match('two').text(), 'two two', here + 'two single matches')
+  t.equal(doc.match('one').text(), 'one, One,', here + 'two more single matches')
+
+  t.equal(doc.match('one two').text(), 'one, two One, two,', here + 'two multi-matches')
+
+  // t.equal(doc.not('two').out(), 'one, three. One, four?', '.not() okay') // ❌
+
+  // t.equal(doc.match('.').out(), str, 'every word') // ❌
+  t.end()
+})
+
+test('pre/post concat', function (t) {
+  const doc = nlp(`Getting ready for whacking day? What's whacking day?`)
+  doc.post(' ', true)
+  t.equal(doc.text(), `Getting ready for whacking day?  What's whacking day? `, here + 'concat')
+  doc.trim()
+  t.equal(doc.text(), `Getting ready for whacking day?  What's whacking day?`)
+  t.end()
+})
+
+test('preserve non-breaking spaces', function (t) {
+  const inputs = [
+    'one\u00a0two\u00a0three.',
+    'One sentence.\u00a0Next sentence.',
+    '\u00a0one\u00a0two\u00a0',
+    'I .\u00a0.\u00a0. maybe.',
+    '「はい。」\u00a0「いいえ。」',
+    'one\ttwo three.',
+  ]
+  inputs.forEach(str => {
+    const doc = nlp(str)
+    const ascii = nlp(str.replace(/\u00a0/g, ' '))
+    t.equal(doc.text(), str, here + 'preserve original whitespace')
+    t.deepEqual(
+      doc.out('array').map(s => s.replace(/\u00a0/g, ' ')),
+      ascii.out('array'),
+      here + 'same sentence boundaries'
+    )
+  })
+  t.end()
+})
+
+test('non-breaking space terms and offsets', function (t) {
+  const str = '\u00a0One\u00a0two.\u00a0Three\u00a0four.\u00a0'
+  const doc = nlp(str)
+  const json = doc.json({ offset: true })
+  t.deepEqual(json.map(s => s.text), ['One\u00a0two.', 'Three\u00a0four.'], here + 'sentence text')
+  t.deepEqual(json.map(s => s.offset), [
+    { index: 0, start: 1, length: 8 },
+    { index: 2, start: 10, length: 11 },
+  ], here + 'sentence offsets')
+  t.deepEqual(json.map(s => s.terms.map(term => term.index)), [
+    [[0, 0], [0, 1]],
+    [[1, 0], [1, 1]],
+  ], here + 'term indices')
+  t.equal(doc.match('three four').text(), 'Three\u00a0four.', here + 'match across non-breaking space')
+  t.equal(nlp('old\u00a0in').text('normal'), 'old in', here + 'explicit normalized output')
+  t.equal(nlp('Miami, FL\u00a033178').has('fl 33178'), true, here + 'split non-breaking space')
+  t.end()
+})