diff --git a/src/1-one/match/methods/parseMatch/02-parseToken.js b/src/1-one/match/methods/parseMatch/02-parseToken.js new file mode 100644 index 0000000..07252d8 --- /dev/null +++ b/src/1-one/match/methods/parseMatch/02-parseToken.js @@ -0,0 +1,237 @@ +const hasMinMax = /\{([0-9]+)?(, *[0-9]*)?\}/ +const andSign = /&&/ +// const hasDash = /\p{Letter}[-–—]\p{Letter}/u +const captureName = /^<\s*([^\s>]+)\s*>/ +/* break-down a match expression into this: +{ + word:'', + tag:'', + regex:'', + + start:false, + end:false, + negative:false, + anything:false, + greedy:false, + optional:false, + + named:'', + choices:[], +} +*/ +const titleCase = str => str.charAt(0).toUpperCase() + str.substring(1) +const end = (str) => str.charAt(str.length - 1) +const start = (str) => str.charAt(0) +const stripStart = (str) => str.substring(1) +const stripEnd = (str) => str.substring(0, str.length - 1) + +const stripBoth = function (str) { + str = stripStart(str) + str = stripEnd(str) + return str +} +// +const parseToken = function (w, opts, aliases) { + const obj = {} + //collect any flags (do it twice) + for (let i = 0; i < 2; i += 1) { + //end-flag + if (end(w) === '$') { + obj.end = true + w = stripEnd(w) + } + //front-flag + if (start(w) === '^') { + obj.start = true + w = stripStart(w) + } + if (end(w) === '?') { + obj.optional = true + w = stripEnd(w) + } + //capture group (this one can span multiple-terms) + if (start(w) === '[' || end(w) === ']') { + obj.group = null + if (start(w) === '[') { + obj.groupStart = true + } + if (end(w) === ']') { + obj.groupEnd = true + } + w = w.replace(/^\[/, '') + w = w.replace(/\]$/, '') + // Use capture group name + if (start(w) === '<') { + const res = captureName.exec(w) + if (!res) { + throw new SyntaxError('Invalid named capture: expected [ ...]') + } + if (res.length >= 2) { + obj.group = res[1] + w = w.replace(res[0], '') + } + } + } + //back-flags + if (end(w) === '+') { + obj.greedy = true + w = stripEnd(w) + } + if (w !== '*' && end(w) === '*' && w !== '\\*') { + obj.greedy = true + w = stripEnd(w) + } + if (start(w) === '!') { + obj.negative = true + // obj.optional = true + w = stripStart(w) + } + //soft-match + if (start(w) === '~' && end(w) === '~' && w.length > 2) { + w = stripBoth(w) + obj.fuzzy = true + obj.min = opts.fuzzy ?? 0.85 + if (/\(/.test(w) === false) { + obj.word = w + return obj + } + } + + //regex + if (start(w) === '/' && end(w) === '/') { + w = stripBoth(w) + if (opts.caseSensitive) { + obj.use = 'text' + } + obj.regex = new RegExp(w) //potential vuln - security/detect-non-literal-regexp + return obj + } + + // support foo{1,9} + if (hasMinMax.test(w) === true) { + w = w.replace(hasMinMax, (_a, b, c) => { + if (c === undefined) { + // '{3}' Exactly three times + obj.min = Number(b) + obj.max = Number(b) + } else { + c = c.replace(/, */, '') + if (b === undefined) { + // '{,9}' implied zero min + obj.min = 0 + obj.max = Number(c) + } else { + // '{2,4}' Two to four times + obj.min = Number(b) + // '{3,}' Three or more times + obj.max = Number(c || 999) + } + } + // use same method as '+' + obj.greedy = true + // 0 as min means the same as '?' + if (!obj.min) { + obj.optional = true + } + return '' + }) + } + + //wrapped-flags + if (start(w) === '(' && end(w) === ')') { + // support (one && two) + if (andSign.test(w)) { + obj.choices = w.split(andSign) + obj.operator = 'and' + } else { + obj.choices = w.split('|') + obj.operator = 'or' + } + //remove '(' and ')' + obj.choices[0] = stripStart(obj.choices[0]) + const last = obj.choices.length - 1 + obj.choices[last] = stripEnd(obj.choices[last]) + // clean up the results + obj.choices = obj.choices.map(s => s.trim()) + obj.choices = obj.choices.filter(s => s) + //recursion alert! + obj.choices = obj.choices.map(str => { + return str.split(/ /g).map(s => parseToken(s, opts, aliases)) + }) + w = '' + } + + //root/sense overloaded + if (start(w) === '{' && end(w) === '}') { + w = stripBoth(w) + // obj.sense = w + obj.root = w + if (/\//.test(w)) { + const split = obj.root.split(/\//) + obj.root = split[0] + obj.pos = split[1] + if (obj.pos === 'adj') { + obj.pos = 'Adjective' + } + // titlecase + obj.pos = obj.pos.charAt(0).toUpperCase() + obj.pos.substr(1).toLowerCase() + // add sense-number too + if (split[2] !== undefined) { + obj.sense = split[2] + } + } + return obj + } + //chunks + if (start(w) === '<' && end(w) === '>') { + w = stripBoth(w) + obj.chunk = titleCase(w) + obj.greedy = true + return obj + } + if (start(w) === '%' && end(w) === '%') { + w = stripBoth(w) + obj.switch = w + return obj + } + } + //do the actual token content + if (start(w) === '#') { + const name = stripStart(w) + obj.tag = aliases?.[name] + if (!obj.tag) { + obj.tag = titleCase(name) + obj.tag = aliases?.[obj.tag] || obj.tag + } + return obj + } + //dynamic function on a term object + if (start(w) === '@') { + obj.method = stripStart(w) + return obj + } + if (w === '.') { + obj.anything = true + return obj + } + //support alone-astrix + if (w === '*') { + obj.anything = true + obj.greedy = true + obj.optional = true + return obj + } + if (w) { + //somehow handle encoded-chars? + w = w.replace('\\*', '*') + w = w.replace('\\.', '.') + if (opts.caseSensitive) { + obj.use = 'text' + } else { + w = w.toLowerCase() + } + obj.word = w + } + return obj +} +export default parseToken diff --git a/tests/01-one/match/encoding.test.js b/tests/01-one/match/encoding.test.js new file mode 100644 index 0000000..84bc1a1 --- /dev/null +++ b/tests/01-one/match/encoding.test.js @@ -0,0 +1,52 @@ +import test from 'tape' +import nlp from '../../lib/one.js' +const here = '[one/match/encoding] ' + +test('reserved-word-in-src:', function (t) { + const r = nlp('buy eggs constructor yeah prototype') + t.equal(r.has(`backburner`), false, here + 'single') + t.equal(r.has(`#Foo`), false, here + 'tag') + t.equal(r.has(`(upcoming|backburner)`), false, here + 'anyOf') + t.equal(r.has(`lala+`), false, here + 'manyOf') + t.equal(r.has(`nword{2,4}`), false, here + 'someOf') + t.end() +}) + +test('reserved-word-in-match:', function (t) { + const r = nlp('fo foo fulala repeat') + t.equal(r.has(`constructor`), false, here + 'single') + t.equal(r.has(`#constructor`), false, here + 'tag') + t.equal(r.has(`(upcoming|constructor)`), false, here + 'anyOf') + t.equal(r.has(`constructor+`), false, here + 'manyOf') + t.equal(r.has(`constructor{2,4}`), false, here + 'someOf') + t.end() +}) + +test('test-infinite-loop', function (t) { + const weirdDoc = nlp('^ ? * . + $') + weirdDoc.match('is?') + weirdDoc.match('.?') + weirdDoc.match('*') + weirdDoc.match('.+') + weirdDoc.match('+') + weirdDoc.match('?') + weirdDoc.match('.') + weirdDoc.match('? * . +') + weirdDoc.not('?') + weirdDoc.not('*') + weirdDoc.not('^') + weirdDoc.not('$') + weirdDoc.not('+') + weirdDoc.not('? * . +') + t.ok(true, here + 'didnt regress') + + let str = 'And you are?. Marshal' + let have = nlp(str).all().out() + t.equal(have, str, here + 'regression #1') + + str = `- where is she.Oh. you guys don't know?` + have = nlp(str).all().out() + t.equal(have, str, here + 'regression #2') + + t.end() +})