diff --git a/src/1-one/lexicon/compute/single-word.js b/src/1-one/lexicon/compute/single-word.js new file mode 100644 index 0000000..9d79fa9 --- /dev/null +++ b/src/1-one/lexicon/compute/single-word.js @@ -0,0 +1,42 @@ +const prefix = /^(under|over|mis|re|un|dis|semi|pre|post)-?/ +// anti|non|extra|inter|intra|over +const allowPrefix = new Set(['Verb', 'Infinitive', 'PastTense', 'Gerund', 'PresentTense', 'Adjective', 'Participle']) + +// tag any words in our lexicon +const checkLexicon = function (terms, i, world) { + const { model, methods } = world + // const fastTag = methods.one.fastTag + const setTag = methods.one.setTag + const { lexicon } = model.one + + // basic lexicon lookup + const t = terms[i] + const word = t.machine || t.normal + // normal lexicon lookup + if (lexicon[word] !== undefined && Object.hasOwn(lexicon, word)) { + setTag([t], lexicon[word], world, false, '1-lexicon') + return true + } + // lookup aliases in the lexicon + if (t.alias) { + const found = t.alias.find(str => Object.hasOwn(lexicon, str)) + if (found) { + setTag([t], lexicon[found], world, false, '1-lexicon-alias') + return true + } + } + // prefixing for verbs/adjectives + if (prefix.test(word) === true) { + const stem = word.replace(prefix, '') + if (Object.hasOwn(lexicon, stem) && stem.length > 3) { + // only allow prefixes for verbs/adjectives + if (allowPrefix.has(lexicon[stem])) { + // console.log('->', word, stem, lexicon[stem]) + setTag([t], lexicon[stem], world, false, '1-lexicon-prefix') + return true + } + } + } + return null +} +export default checkLexicon diff --git a/tests/one/lexicon/core-lexicon.test.js b/tests/one/lexicon/core-lexicon.test.js new file mode 100644 index 0000000..8d24fdf --- /dev/null +++ b/tests/one/lexicon/core-lexicon.test.js @@ -0,0 +1,86 @@ +import test from 'tape' +import nlp from '../_lib.js' +const here = '[one/lexicon] ' + +test('addWords side-load:', function (t) { + let lex = { + bigg: 'Size', + 'bigg apple': 'Town', + } + const _nlp = nlp //.fork() + let doc = _nlp('it was bigg') + t.equal(doc.has('(#Size|#Town)'), false, here + 'none-on-init') + + doc = _nlp('it was bigg', lex) + t.equal(doc.has('#Size'), true, here + 'single-word') + doc = _nlp('the bigg apple', lex) + t.equal(doc.has('#Town'), true, here + 'found-multi-word') + t.equal(doc.has('#Size'), false, here + 'no-single-word') + + doc = _nlp('it was bigg') + t.equal(doc.has('#Size'), true, here + 'lex-persists') + + lex = { + bigg: undefined, + 'bigg apple': undefined, + } + doc = _nlp('it was bigg', lex) + t.equal(doc.has('#Size'), false, here + 'lex-removed') + doc = _nlp('the bigg apple', lex) + t.equal(doc.has('#Town'), false, here + 'multi-lex-removed') + + t.end() +}) + +test('lexicon compute:', function (t) { + const words = { + 'captain obvious': 'Captain', + } + const doc = nlp('it was captain obvious', words) + const m = doc.match('#Captain+') + t.equal(m.text(), 'captain obvious', here + 'multi-word') + t.end() +}) + +test('tricky lexicon:', function (t) { + const lexicon = { + 'bed bath and beyond': 'Organization', + } + const _nlp = nlp //.fork() + let r = _nlp('shopping at Bed Bath and Beyond, the store', lexicon) + let str = r.match('#Organization+').out('normal') + t.equal(str, 'bed bath and beyond', here + 'four-word') + + r = _nlp('shopping at Bed Bath and-beyond the store', lexicon) + str = r.match('#Organization+').out('normal') + t.equal(str, 'bed bath and beyond', here + 'partially-hyphenated-word') + + r = _nlp('shopping at Bed-bath and-beyond the store', lexicon) + str = r.match('#Organization+').out('normal') + t.equal(str, 'bed bath and beyond', here + 'many-hyphenated-word') + t.end() +}) + +test('apostrophe lexicon:', function (t) { + const lex = { + "queen anne's lace": 'Flower', + "applebee's": 'Restaurant', + } + let doc = nlp(`i went to applebee's for dinner`, lex) + t.equal(doc.has(`#Restaurant`), true, here + 'lexicon w/ apostrophe') + + doc = nlp(`Queen Anne's lace`, lex) + t.equal(doc.has(`#Flower`), true, here + 'multi lexicon w/ apostrophe') + t.end() +}) + +test('long lexicon:', function (t) { + nlp.addWords({ 'new york yankees are cool and not bad': 'Long' }) + let doc = nlp('the new york yankees are cool and smart') + t.equal(doc.has('#Long'), false, here + 'missed-long') + + doc = nlp('the new york yankees are cool and not bad') + t.equal(doc.has('#Long'), true, here + 'found-long') + + t.end() +})