diff --git a/src/1-one/match/methods/parseMatch/03-splitHyphens.js b/src/1-one/match/methods/parseMatch/03-splitHyphens.js new file mode 100644 index 0000000..4ebacc3 --- /dev/null +++ b/src/1-one/match/methods/parseMatch/03-splitHyphens.js @@ -0,0 +1,25 @@ +const hasDash = /[a-z0-9][-–—][a-z]/i + +// match 're-do' -> ['re','do'] +const splitHyphens = function (regs, world) { + const prefixes = world.model.one.prefixes + for (let i = regs.length - 1; i >= 0; i -= 1) { + const reg = regs[i] + if (reg.word && hasDash.test(reg.word)) { + let words = reg.word.split(/[-–—]/g) + // don't split 're-cycle', etc + if (Object.hasOwn(prefixes, words[0])) { + continue + } + words = words.filter(w => w).reverse() + regs.splice(i, 1) + words.forEach(w => { + const obj = { ...reg } + obj.word = w + regs.splice(i, 0, obj) + }) + } + } + return regs +} +export default splitHyphens \ No newline at end of file diff --git a/tests/one/tokenize/hyphen-matrix.test.js b/tests/one/tokenize/hyphen-matrix.test.js new file mode 100644 index 0000000..605f196 --- /dev/null +++ b/tests/one/tokenize/hyphen-matrix.test.js @@ -0,0 +1,88 @@ +import test from 'tape' +import nlp from '../_lib.js' +const here = '[one/hyphen-matrix] ' + +test('hyphen-input', (t) => { + const doc = nlp(`before follow-up after`) + t.equal(doc.has('follow-up'), true, here + 'hyphen -> follow-up') + t.equal(doc.has('follow up'), true, here + 'hyphen -> follow up') + t.equal(doc.has('followup'), false, here + 'hyphen -> followup')//would be nice + t.end() +}) + +test('no-hyphen-input', (t) => { + const doc = nlp(`before follow up after`) + t.equal(doc.has('follow-up'), true, here + 'no-hyphen -> follow-up') + t.equal(doc.has('follow up'), true, here + 'no-hyphen ->follow up') + t.equal(doc.has('followup'), false, here + 'no-hyphen ->followup') + t.end() +}) + +test('compound-hyphen-input', (t) => { + const doc = nlp(`before followup after`) + t.equal(doc.has('follow-up'), false, here + 'compound -> follow-up')//maybe? + t.equal(doc.has('follow up'), false, here + 'compound -> follow up') + t.equal(doc.has('followup'), true, here + 'compound -> followup') + t.end() +}) + + +// ================== + +test('hyphen-skipping', (t) => { + const doc = nlp(`before super-cool after`) + t.equal(doc.has('before super'), true, here + '[hyphen] before') + t.equal(doc.has('before super cool'), true, here + '[hyphen] before-mid') + t.equal(doc.has('cool after'), true, here + '[hyphen] after') + t.equal(doc.has('super cool after'), true, here + '[hyphen] after-mid') + // + t.equal(doc.has('before after'), false, here + '[hyphen] no-jump') + t.equal(doc.has('before super after'), false, here + '[hyphen] no-mid-jump') + t.equal(doc.has('before cool after'), false, here + '[hyphen] no-mid-2-jump') + t.end() +}) + + +test('match-dash', function (t) { + const arr = [ + 're-purpose', + 'co-opting', + 'mis-information', + 'proto-plasmic', + 'counter-argument', + 'soft-sell', + 'big-news', + 'do-over', + 'over-the-top', + 'larger-than-life', + 're-zoning-laws', + 'inter-sectional', + 'counter-argument', + 're-purpose itself', + 'full-enough tank', + 'the size-difference', + 'counter-balance', + 'score was 10-2', + 'Nobel Prize–winning', + 'take-down ', + 'the non-player-character', + 'load-bearing walls', + 'this clearly-impossible story', + 'beautiful-looking flowers ', + 'community-based education', + 'Mother-in-law', + 'Master-at-arms', + 'Editor-in-chief', + 'Ten-year-old', + 'Factory-made', + 'Twelve-pack', + 'fifty-six bottles', + 'a 10-minute speech', + 'self-serve', + 'non-sequitur' + ] + arr.forEach(str => { + t.equal(nlp(str).has(str), true, here + '[dash] ' + str) + }) + t.end() +})