Split hyphens in a match the same way the tokenizer does
This commit is contained in:
1 parent
a85b530171
commit
1440e67433
2 files changed
+10
-8
No files matched your search
@@ -1,16 +1,12 @@
|
||||
const hasDash = /[a-z0-9][-–—][a-z]/i
|
||||
import { hasHyphen } from '../../../tokenize/methods/02-terms/01-hyphens.js'
|
||||
|
||||
// match 're-do' -> ['re','do']
|
||||
// split them the same way the tokenizer does - 'x-ray' stays one word
|
||||
const splitHyphens = function (regs, world) {
|
||||
const prefixes = world.model.one.prefixes
|
||||
for (let i = regs.length - 1; i >= 0; i -= 1) {
|
||||
const reg = regs[i]
|
||||
if (reg.word && hasDash.test(reg.word)) {
|
||||
if (reg.word && hasHyphen(reg.word, world.model)) {
|
||||
let words = reg.word.split(/[-–—]/g)
|
||||
// don't split 're-cycle', etc
|
||||
if (Object.hasOwn(prefixes, words[0])) {
|
||||
continue
|
||||
}
|
||||
words = words.filter(w => w).reverse()
|
||||
regs.splice(i, 1)
|
||||
words.forEach(w => {
|
||||
|
||||
@@ -79,7 +79,13 @@ test('match-dash', function (t) {
|
||||
'fifty-six bottles',
|
||||
'a 10-minute speech',
|
||||
'self-serve',
|
||||
'non-sequitur'
|
||||
'non-sequitur',
|
||||
'an x-ray was done',
|
||||
'the x-ray-machine broke',
|
||||
'she had covid-19',
|
||||
'a t-shirt',
|
||||
'a flower-like shape',
|
||||
'the president-elect',
|
||||
]
|
||||
arr.forEach(str => {
|
||||
t.equal(nlp(str).has(str), true, here + '[dash] ' + str)
|
||||
|
||||
Reference in new issue
Block a user