Split hyphens in a match the same way the tokenizer does

This commit is contained in:
dualfroz committed 2026-10-03 03:09:19 +02:00
1 parent a85b530171
commit 1440e67433
2 files changed
+10 -8

No files matched your search

@@ -1,16 +1,12 @@
const hasDash = /[a-z0-9][-–—][a-z]/i import { hasHyphen } from '../../../tokenize/methods/02-terms/01-hyphens.js'
// match 're-do' -> ['re','do'] // match 're-do' -> ['re','do']
// split them the same way the tokenizer does - 'x-ray' stays one word
const splitHyphens = function (regs, world) { const splitHyphens = function (regs, world) {
const prefixes = world.model.one.prefixes
for (let i = regs.length - 1; i >= 0; i -= 1) { for (let i = regs.length - 1; i >= 0; i -= 1) {
const reg = regs[i] const reg = regs[i]
if (reg.word && hasDash.test(reg.word)) { if (reg.word && hasHyphen(reg.word, world.model)) {
let words = reg.word.split(/[-–—]/g) let words = reg.word.split(/[-–—]/g)
// don't split 're-cycle', etc
if (Object.hasOwn(prefixes, words[0])) {
continue
}
words = words.filter(w => w).reverse() words = words.filter(w => w).reverse()
regs.splice(i, 1) regs.splice(i, 1)
words.forEach(w => { words.forEach(w => {
+7 -1
View File
@@ -79,7 +79,13 @@ test('match-dash', function (t) {
'fifty-six bottles', 'fifty-six bottles',
'a 10-minute speech', 'a 10-minute speech',
'self-serve', 'self-serve',
'non-sequitur' 'non-sequitur',
'an x-ray was done',
'the x-ray-machine broke',
'she had covid-19',
'a t-shirt',
'a flower-like shape',
'the president-elect',
] ]
arr.forEach(str => { arr.forEach(str => {
t.equal(nlp(str).has(str), true, here + '[dash] ' + str) t.equal(nlp(str).has(str), true, here + '[dash] ' + str)