diff --git a/plugins/dates/tests/regressions.test.js b/plugins/dates/tests/regressions.test.js index 8dd9ec3..13fec18 100644 --- a/plugins/dates/tests/regressions.test.js +++ b/plugins/dates/tests/regressions.test.js @@ -167,11 +167,3 @@ test('a range does not stop other dates from splitting', (t) => { t.equal(res.length, 3, 'three dates found') t.end() }) - -test('singular possessive durations', (t) => { - t.equal(getStart("in a month's time"), '2026-08-11T00:00:00.000-04:00', "a month's time") - t.equal(getStart("in a week's time"), '2026-07-18T00:00:00.000-04:00', "a week's time") - t.equal(getStart("in a year's time"), '2027-07-11T00:00:00.000-04:00', "a year's time") - t.equal(getStart("in two weeks' time"), '2026-07-25T00:00:00.000-04:00', "two weeks' time") - t.end() -}) diff --git a/src/1-one/output/api/_text.js b/src/1-one/output/api/_text.js new file mode 100644 index 0000000..ac1c47b --- /dev/null +++ b/src/1-one/output/api/_text.js @@ -0,0 +1,107 @@ +// Walk the trailing punctuation run once, rather than retrying at every offset. +const endPunctuation = /[,:;)\]*.?~!\u0022\uFF02\u201D\u2019\u00BB\u203A\u2032\u2033\u2034\u301E\u00B4—-]/ +const trimEnd = str => { + let end = str.length + while (end > 0 && endPunctuation.test(str[end - 1])) end-- + return str.slice(0, end) +} +const trimStart = + /^[(['"*~\uFF02\u201C\u2018\u201F\u201B\u201E\u2E42\u201A\u00AB\u2039\u2035\u2036\u2037\u301D\u0060\u301F]+/ + +const punctToKill = /[,:;)('"\u201D\]]/ +const isHyphen = /^[-–—]$/ +const hasSpace = / / + +const textFromTerms = function (terms, opts, keepSpace = true) { + let txt = '' + terms.forEach(t => { + let pre = t.pre || '' + let post = t.post || '' + if (opts.punctuation === 'some') { + pre = pre.replace(trimStart, '') + // replace a hyphen with a space + if (isHyphen.test(post)) { + post = ' ' + } + post = post.replace(punctToKill, '') + // cleanup exclamations + post = post.replace(/\?!+/, '?') + post = post.replace(/!+/, '!') + post = post.replace(/\?+/, '?') + // kill elipses + post = post.replace(/\.{2,}/, '') + // kill abbreviation periods + if (t.tags.has('Abbreviation')) { + post = post.replace(/\./, '') + } + } + if (opts.whitespace === 'some') { + pre = pre.replace(/\s/, '') //remove pre-whitespace + post = post.replace(/\s+/, ' ') //replace post-whitespace with a space + } + if (!opts.keepPunct) { + pre = pre.replace(trimStart, '') + if (post === '-') { + post = ' ' + } else { + post = trimEnd(post) + } + } + // grab the correct word format + let word = t[opts.form || 'text'] || t.normal || '' + if (opts.form === 'implicit') { + word = t.implicit || t.text + } + if (opts.form === 'root' && t.implicit) { + word = t.root || t.implicit || t.normal + } + // add an implicit space, for contractions + if ((opts.form === 'machine' || opts.form === 'implicit' || opts.form === 'root') && t.implicit) { + if (!post || !hasSpace.test(post)) { + post += ' ' + } + } + txt += pre + word + post + }) + if (keepSpace === false) { + txt = txt.trim() + } + if (opts.lowerCase === true) { + txt = txt.toLowerCase() + } + return txt +} + +const textFromDoc = function (docs, opts) { + let text = '' + if (!docs || !docs[0] || !docs[0][0]) { + return text + } + for (let i = 0; i < docs.length; i += 1) { + // middle + text += textFromTerms(docs[i], opts, true) + } + if (!opts.keepSpace) { + text = text.trim() + } + if (opts.keepEndPunct === false) { + // don't remove ':)' etc + if (!docs[0][0].tags.has('Emoticon')) { + text = text.replace(trimStart, '') + } + // remove ending periods + const last = docs.at(-1) + if (!last.at(-1).tags.has('Emoticon')) { + text = trimEnd(text) + } + // kill end quotations + if (text.endsWith(`'`) && !text.endsWith(`s'`)) { + text = text.replace(/'/, '') + } + } + if (opts.cleanWhitespace === true) { + text = text.trim() + } + return text +} +export { textFromDoc, textFromTerms } diff --git a/src/1-one/output/api/text.js b/src/1-one/output/api/text.js new file mode 100644 index 0000000..dc39338 --- /dev/null +++ b/src/1-one/output/api/text.js @@ -0,0 +1,39 @@ +import { textFromDoc } from './_text.js' +import fmts from './_fmts.js' + +const isObject = val => { + return Object.prototype.toString.call(val) === '[object Object]' +} + +export default { + /** */ + text: function (fmt) { + let opts = {} + if (fmt && typeof fmt === 'string' && Object.hasOwn(fmts, fmt)) { + opts = { ...fmts[fmt] } + } else if (fmt && isObject(fmt)) { + opts = { ...fmt } //todo: fixme + } + // is it a full document? + if (opts.keepSpace === undefined && !this.isFull()) { + // + opts.keepSpace = false + } + if (opts.keepEndPunct === undefined && this.pointer) { + const ptr = this.pointer[0] + if (ptr?.[1]) { + opts.keepEndPunct = false + } else { + opts.keepEndPunct = true + } + } + // set defaults + if (opts.keepPunct === undefined) { + opts.keepPunct = true + } + if (opts.keepSpace === undefined) { + opts.keepSpace = true + } + return textFromDoc(this.docs, opts) + }, +} diff --git a/tests/one/change/whitespace.test.js b/tests/one/change/whitespace.test.js new file mode 100644 index 0000000..e9dd214 --- /dev/null +++ b/tests/one/change/whitespace.test.js @@ -0,0 +1,76 @@ +import test from 'tape' +import nlp from '../_lib.js' +const here = '[one/whitespace] ' + +test('whitespace-out', function (t) { + const str = 'one, two three. One, two, four?' + const doc = nlp(str) + + t.equal(doc.out(), str, here + 'original-okay') + + // some phrases, but full-phrases + t.equal(doc.eq(0).text(), 'one, two three.', here + '.eq(0) okay') + t.equal(doc.eq(1).text(), 'One, two, four?', here + '.eq(1) okay') + + t.equal(doc.match('four').text(), 'four', here + 'one match') + + t.equal(doc.match('two').text(), 'two two', here + 'two single matches') + t.equal(doc.match('one').text(), 'one, One,', here + 'two more single matches') + + t.equal(doc.match('one two').text(), 'one, two One, two,', here + 'two multi-matches') + + // t.equal(doc.not('two').out(), 'one, three. One, four?', '.not() okay') // ❌ + + // t.equal(doc.match('.').out(), str, 'every word') // ❌ + t.end() +}) + +test('pre/post concat', function (t) { + const doc = nlp(`Getting ready for whacking day? What's whacking day?`) + doc.post(' ', true) + t.equal(doc.text(), `Getting ready for whacking day? What's whacking day? `, here + 'concat') + doc.trim() + t.equal(doc.text(), `Getting ready for whacking day? What's whacking day?`) + t.end() +}) + +test('preserve non-breaking spaces', function (t) { + const inputs = [ + 'one\u00a0two\u00a0three.', + 'One sentence.\u00a0Next sentence.', + '\u00a0one\u00a0two\u00a0', + 'I .\u00a0.\u00a0. maybe.', + '「はい。」\u00a0「いいえ。」', + 'one\ttwo three.', + ] + inputs.forEach(str => { + const doc = nlp(str) + const ascii = nlp(str.replace(/\u00a0/g, ' ')) + t.equal(doc.text(), str, here + 'preserve original whitespace') + t.deepEqual( + doc.out('array').map(s => s.replace(/\u00a0/g, ' ')), + ascii.out('array'), + here + 'same sentence boundaries' + ) + }) + t.end() +}) + +test('non-breaking space terms and offsets', function (t) { + const str = '\u00a0One\u00a0two.\u00a0Three\u00a0four.\u00a0' + const doc = nlp(str) + const json = doc.json({ offset: true }) + t.deepEqual(json.map(s => s.text), ['One\u00a0two.', 'Three\u00a0four.'], here + 'sentence text') + t.deepEqual(json.map(s => s.offset), [ + { index: 0, start: 1, length: 8 }, + { index: 2, start: 10, length: 11 }, + ], here + 'sentence offsets') + t.deepEqual(json.map(s => s.terms.map(term => term.index)), [ + [[0, 0], [0, 1]], + [[1, 0], [1, 1]], + ], here + 'term indices') + t.equal(doc.match('three four').text(), 'Three\u00a0four.', here + 'match across non-breaking space') + t.equal(nlp('old\u00a0in').text('normal'), 'old in', here + 'explicit normalized output') + t.equal(nlp('Miami, FL\u00a033178').has('fl 33178'), true, here + 'split non-breaking space') + t.end() +})