Split hyphens in a match the same way the tokenizer does
This commit is contained in:
1 parent
a85b530171
commit
1440e67433
2 files changed
+10
-8
No files matched your search
@@ -1,16 +1,12 @@
|
|||||||
const hasDash = /[a-z0-9][-–—][a-z]/i
|
import { hasHyphen } from '../../../tokenize/methods/02-terms/01-hyphens.js'
|
||||||
|
|
||||||
// match 're-do' -> ['re','do']
|
// match 're-do' -> ['re','do']
|
||||||
|
// split them the same way the tokenizer does - 'x-ray' stays one word
|
||||||
const splitHyphens = function (regs, world) {
|
const splitHyphens = function (regs, world) {
|
||||||
const prefixes = world.model.one.prefixes
|
|
||||||
for (let i = regs.length - 1; i >= 0; i -= 1) {
|
for (let i = regs.length - 1; i >= 0; i -= 1) {
|
||||||
const reg = regs[i]
|
const reg = regs[i]
|
||||||
if (reg.word && hasDash.test(reg.word)) {
|
if (reg.word && hasHyphen(reg.word, world.model)) {
|
||||||
let words = reg.word.split(/[-–—]/g)
|
let words = reg.word.split(/[-–—]/g)
|
||||||
// don't split 're-cycle', etc
|
|
||||||
if (Object.hasOwn(prefixes, words[0])) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
words = words.filter(w => w).reverse()
|
words = words.filter(w => w).reverse()
|
||||||
regs.splice(i, 1)
|
regs.splice(i, 1)
|
||||||
words.forEach(w => {
|
words.forEach(w => {
|
||||||
|
|||||||
@@ -79,7 +79,13 @@ test('match-dash', function (t) {
|
|||||||
'fifty-six bottles',
|
'fifty-six bottles',
|
||||||
'a 10-minute speech',
|
'a 10-minute speech',
|
||||||
'self-serve',
|
'self-serve',
|
||||||
'non-sequitur'
|
'non-sequitur',
|
||||||
|
'an x-ray was done',
|
||||||
|
'the x-ray-machine broke',
|
||||||
|
'she had covid-19',
|
||||||
|
'a t-shirt',
|
||||||
|
'a flower-like shape',
|
||||||
|
'the president-elect',
|
||||||
]
|
]
|
||||||
arr.forEach(str => {
|
arr.forEach(str => {
|
||||||
t.equal(nlp(str).has(str), true, here + '[dash] ' + str)
|
t.equal(nlp(str).has(str), true, here + '[dash] ' + str)
|
||||||
|
|||||||
Reference in new issue
Block a user