diff --git a/tests/two/regression/regex-boundaries.test.js b/tests/two/regression/regex-boundaries.test.js new file mode 100644 index 0000000..d8139e3 --- /dev/null +++ b/tests/two/regression/regex-boundaries.test.js @@ -0,0 +1,118 @@ +import test from 'tape' +import nlp from '../_lib.js' + +const wrappers = [ + ['', ''], + ['"', '"'], + ["'", "'"], + ['“', '”'], + ['‘', '’'], + ['(', ')'], +] + +test('generated email and URL boundaries preserve text and selections', t => { + const emails = [] + for (const local of ['alice', 'first-last', 'first.last+news']) { + for (const domain of ['example.com', 'my-site.info', 'sub.my-site.technology']) { + emails.push(`${local}@${domain}`) + } + } + const urls = [] + for (const base of ['x.io', 'my-site.com', 'https://my-site.xyz', 'www.my-site.technology']) { + for (const tail of ['', '/a-b', '?q=yes', '#section', ':8080/path?q=yes#part']) urls.push(base + tail) + } + const failures = [] + let count = 0 + for (const [tag, values] of [ + ['Email', emails], + ['Url', urls], + ]) { + for (const value of values) { + for (const [open, close] of wrappers) { + for (const separator of [' ', '\n', '\r\n']) { + for (const ending of [',', '.']) { + const text = `See ${open}${value}${close}${ending}${separator}Thanks.` + const doc = nlp(text) + const found = doc.match(`#${tag}`) + count++ + if (doc.text() !== text || found.terms().length !== 1 || found.text('normal') !== value) { + failures.push({ text, output: doc.text(), selected: found.text('normal') }) + } + } + } + } + } + } + t.deepEqual(failures, [], `${count} generated address contexts preserve text and isolate the complete address`) + t.end() +}) + +test('generated punctuation contexts preserve Unicode and are stable when normalized', t => { + const failures = [] + let count = 0 + for (const word of ['alpha', 'café', '𐐀word', '東京', 'word😀word']) { + for (const [open, close] of wrappers) { + for (const prefix of ['', '!!!', ' ', '😀']) { + for (const suffix of ['', '?!', '…', '\n', ' ']) { + const text = prefix + open + word + close + suffix + const reconstructed = nlp(text).text() + if (reconstructed !== text) failures.push({ text, reconstructed }) + count++ + } + } + } + } + t.deepEqual(failures, [], `${count} punctuation contexts round-trip code points`) + const normalizationFailures = [] + for (const infix of ['.', '...', ',', '!', '?', '-', '😀']) { + for (const ending of ['', '.', '!!!', '…', ')']) { + const text = `("AlPhA${infix}BeTa${ending}` + const expected = `alpha${infix}beta` + const doc = nlp(text) + const normals = doc.json().flatMap(sentence => sentence.terms.map(term => term.normal)) + doc.normalize() + const normalized = doc.text() + doc.normalize() + if (normals.length !== 1 || normals[0] !== expected || doc.text() !== normalized) { + normalizationFailures.push({ text, expected, normals, normalized, repeated: doc.text() }) + } + } + } + t.deepEqual( + normalizationFailures, + [], + 'term normals preserve internal punctuation and document normalization is idempotent' + ) + t.end() +}) + +test('match groups preserve captures, repetition and surrounding words', t => { + const cases = [ + ['before red after', 'before [(red|blue)] after', 'red'], + ['before blue after', 'before [(red|blue)] after', 'blue'], + ['before red blue after', 'before [(red|blue)+] after', 'red blue'], + ['before after', 'before [(red|blue)?] after', ''], + ['before two words after', 'before [(two words|three words)] after', 'two words'], + ['before three words after', 'before [(two words|three words)] after', 'three words'], + ['before cat after', 'before [(#Noun && cat)] after', 'cat'], + ['before green after', 'before [(!red && !blue)] after', 'green'], + ] + for (const [text, pattern, captured] of cases) { + const doc = nlp(text) + for (const anchored of [pattern, '^' + pattern + '$']) { + const match = doc.match(anchored) + t.equal(match.text(), text, anchored + ' matches the whole phrase') + t.equal(match.groups('choice').text(), captured, 'only the chosen words are captured') + } + } + for (const text of ['before green after', 'before red elsewhere', 'elsewhere red after']) { + t.notOk(nlp(text).has('before (red|blue) after'), 'reject mismatched alternatives or surrounding words') + } + t.notOk(nlp('before red after').has('before !(red|blue) after'), 'negation excludes the alternatives') + t.notOk(nlp('extra before red after').has('^before (red|blue) after$'), 'start anchor excludes preceding words') + t.notOk(nlp('before red after extra').has('^before (red|blue) after$'), 'end anchor excludes following words') + for (const pattern of ['[foo]', '[ nlp.parseMatch(pattern), /Invalid named capture/, 'malformed named capture: ' + pattern) + } + t.end() +})