From 1440e6743356971692377aedc6ff92b5c5cdc280 Mon Sep 17 00:00:00 2001 From: dualfroz Date: Sat, 3 Oct 2026 03:09:19 +0200 Subject: [PATCH] Split hyphens in a match the same way the tokenizer does --- src/1-one/match/methods/parseMatch/03-splitHyphens.js | 10 +++------- tests/one/tokenize/hyphen-matrix.test.js | 8 +++++++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/1-one/match/methods/parseMatch/03-splitHyphens.js b/src/1-one/match/methods/parseMatch/03-splitHyphens.js index 4ebacc3..f4c3248 100644 --- a/src/1-one/match/methods/parseMatch/03-splitHyphens.js +++ b/src/1-one/match/methods/parseMatch/03-splitHyphens.js @@ -1,16 +1,12 @@ -const hasDash = /[a-z0-9][-–—][a-z]/i +import { hasHyphen } from '../../../tokenize/methods/02-terms/01-hyphens.js' // match 're-do' -> ['re','do'] +// split them the same way the tokenizer does - 'x-ray' stays one word const splitHyphens = function (regs, world) { - const prefixes = world.model.one.prefixes for (let i = regs.length - 1; i >= 0; i -= 1) { const reg = regs[i] - if (reg.word && hasDash.test(reg.word)) { + if (reg.word && hasHyphen(reg.word, world.model)) { let words = reg.word.split(/[-–—]/g) - // don't split 're-cycle', etc - if (Object.hasOwn(prefixes, words[0])) { - continue - } words = words.filter(w => w).reverse() regs.splice(i, 1) words.forEach(w => { diff --git a/tests/one/tokenize/hyphen-matrix.test.js b/tests/one/tokenize/hyphen-matrix.test.js index 605f196..a49f4a4 100644 --- a/tests/one/tokenize/hyphen-matrix.test.js +++ b/tests/one/tokenize/hyphen-matrix.test.js @@ -79,7 +79,13 @@ test('match-dash', function (t) { 'fifty-six bottles', 'a 10-minute speech', 'self-serve', - 'non-sequitur' + 'non-sequitur', + 'an x-ray was done', + 'the x-ray-machine broke', + 'she had covid-19', + 'a t-shirt', + 'a flower-like shape', + 'the president-elect', ] arr.forEach(str => { t.equal(nlp(str).has(str), true, here + '[dash] ' + str)