67 lines
3.5 KiB
JavaScript
67 lines
3.5 KiB
JavaScript
import test from 'tape'
|
||
import nlp from '../../lib/one.js'
|
||
const here = '[one/tokenize/unicode] '
|
||
|
||
test('many-unicode', function (t) {
|
||
const str = `✐✠✰❀❐❞❰➀➐➠➰✁✑✡✱❁❑❡❱➁➑➡➱✂✒✢✲❂❒❢❲➂➒➢➲✃✓✣✳❃❓❣❳➃➓➣➳✄✔✤✴❄❔❤❴➄➔➤➴✅✕✥✵❅❕❥❵➅➕➥➵✆✖✦✶❆❖❦❶➆➖➦➶✇✗✧✷❇❗❧❷➇➗➧➷✈✘✨✸❈❘❨❸➈➘➨➸✉✙✩✹❉❙❩❹➉➙➩➹✊✚✪✺❊❚❪❺➊➚➪➺✋✛✫✻❋❛❫❻➋➛➫➻✌✜✬✼❌❜❬❼➌➜➬➼✍✝✭✽❍❝❭❽➍➝➭➽✎✞✮✾❎❞❮❾➎➞➮➾✏✟✯✿❏❜❯❿➏➟➯➿😀😐😠😰🙀😁😑😡😱🙁😂😒😢😲🙂😃😓😣😳🙃😄😔😤😴🙄😅😕😥😵🙅😆😖😦😶🙆😇😗😧😷🙇😈😘😨😸🙈😉😙😩😹🙉😊😚😪😺🙊😋😛😫😻🙋😌😜😬😼🙌😍😝😭😽🙍😎😞😮😾🙎😏😟😯😿🙏,&、*.+-;<:>?=!—\($)%{@}〔〕₠₰₡₱₢₲₣₳₤₴₥₵₦₶₧₷₸₩₹₪₺₫₻€₼₭₽₮₾₯₿` // eslint-dis
|
||
const doc = nlp(str)
|
||
t.equal(doc.text(), str, here + 'identical-text')
|
||
t.equal(doc.length, 1, here + 'one-sentence')
|
||
t.equal(doc.terms().length, 1, here + 'one-term')
|
||
t.end()
|
||
})
|
||
|
||
test('em-dashes', function (t) {
|
||
const str = 'text—text'
|
||
const doc = nlp(str)
|
||
t.equal(doc.text() === str, true, here + 'emdash')
|
||
t.end()
|
||
})
|
||
|
||
// this section is very cursed
|
||
test('zero-width-chars', function (t) {
|
||
//this has a zero-width character
|
||
// eslint-disable-next-line no-irregular-whitespace -- Intentional tokenization fixture.
|
||
const str = `before after`
|
||
const doc = nlp(str)
|
||
t.equal(doc.text(), str, here + 'zero-width passes-through')
|
||
const json = doc.json({ terms: { normal: true } })
|
||
const before = json[0].terms[0]
|
||
t.equal(before.normal, 'before', here + 'normalized-out in json')
|
||
t.equal(before.post, ' ', here + 'normal whitespace in json')
|
||
t.ok(doc.text() !== 'before after', here + 'default text has 0-width-char')
|
||
t.equal(doc.text('normal'), 'before after', here + 'normal text removes 0-width-char')
|
||
t.equal(doc.text('clean'), 'before after', here + 'clean text removes 0-width-char')
|
||
t.equal(doc.text('reduced'), 'before after', here + 'reduced text removes 0-width-char')
|
||
t.equal(doc.text('root'), 'before after', here + 'root text removes 0-width-char')
|
||
t.end()
|
||
})
|
||
|
||
test('fullwidth-ascii', function (t) {
|
||
t.equal(nlp('ＡＢＣ').has('abc'), true, here + 'fullwidth match')
|
||
t.equal(nlp('Ｈｅｌｌｏ ２０２４').out('normal'), 'hello 2024', here + 'fullwidth normal')
|
||
t.equal(nlp('Ｈｅｌｌｏ ２０２４').text(), 'Ｈｅｌｌｏ ２０２４', here + 'fullwidth text unchanged')
|
||
t.end()
|
||
})
|
||
|
||
test('combining-marks', function (t) {
|
||
const arr = [
|
||
['caf\u00e9 au lait'.normalize('NFD'), 'cafe\u0301', ' '],
|
||
['Poincare\u0301', 'Poincare\u0301', ''],
|
||
['E\u0300\u0304 x', 'E\u0300\u0304', ' '],
|
||
['(cafe\u0301)', 'cafe\u0301', ')'],
|
||
['cafe\u0301!', 'cafe\u0301', '!'],
|
||
['\u0928\u092e\u0938\u094d\u0924\u0947 x', '\u0928\u092e\u0938\u094d\u0924\u0947', ' '],
|
||
['\u2764\ufe0f x', '\u2764\ufe0f', ' '],
|
||
]
|
||
arr.forEach(a => {
|
||
const [str, text, post] = a
|
||
const doc = nlp(str)
|
||
const term = doc.json()[0].terms[0]
|
||
t.equal(term.text, text, here + `text "${str}"`)
|
||
t.equal(term.post, post, here + `post "${str}"`)
|
||
t.equal(doc.text(), str, here + `identical-text "${str}"`)
|
||
})
|
||
t.end()
|
||
})
|