Split hyphens in a match the same way the tokenizer does

This commit is contained in:
dualfroz committed 2026-10-03 03:09:19 +02:00
1 parent a85b530171
commit 1440e67433
2 files changed
+10 -8

No files matched your search

@@ -1,16 +1,12 @@
const hasDash = /[a-z0-9][-–—][a-z]/i
import { hasHyphen } from '../../../tokenize/methods/02-terms/01-hyphens.js'
// match 're-do' -> ['re','do']
// split them the same way the tokenizer does - 'x-ray' stays one word
const splitHyphens = function (regs, world) {
const prefixes = world.model.one.prefixes
for (let i = regs.length - 1; i >= 0; i -= 1) {
const reg = regs[i]
if (reg.word && hasDash.test(reg.word)) {
if (reg.word && hasHyphen(reg.word, world.model)) {
let words = reg.word.split(/[-–—]/g)
// don't split 're-cycle', etc
if (Object.hasOwn(prefixes, words[0])) {
continue
}
words = words.filter(w => w).reverse()
regs.splice(i, 1)
words.forEach(w => {