Upstream spencermountain/compromise state of the touched files before 8787708938
This commit is contained in:
upstream committed
1 parent
6528d59d86
commit
b22c9bdc67
3 files changed
+297
No files matched your search
@@ -0,0 +1,61 @@
|
||||
import fastTag from '../_fastTag.js'
|
||||
import fillTags from '../3rd-pass/_fillTags.js'
|
||||
|
||||
const titleCase = /^\p{Lu}[\p{Ll}'’]/u
|
||||
const hasNumber = /[0-9]/
|
||||
const notProper = ['Date', 'Month', 'WeekDay', 'Unit', 'Expression']
|
||||
|
||||
// roman numeral by regex
|
||||
const hasIVX = /[IVX]/ // does it ~look like~ a roman numeral?
|
||||
// quick-version
|
||||
const romanNumeral = /^[IVXLCDM]{2,}$/
|
||||
// https://stackoverflow.com/a/267405/168877
|
||||
const romanNumValid = /^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$/
|
||||
const nope = {
|
||||
li: true,
|
||||
dc: true,
|
||||
md: true,
|
||||
dm: true,
|
||||
ml: true,
|
||||
}
|
||||
|
||||
// if it's a unknown titlecase word, it's a propernoun
|
||||
const checkCase = function (terms, i, model) {
|
||||
const term = terms[i]
|
||||
// assume terms are already indexed
|
||||
term.index ||= [0, 0]
|
||||
const index = term.index[1]
|
||||
const str = term.text || '' //need case info
|
||||
// titlecase and not first word of sentence
|
||||
if (index !== 0 && titleCase.test(str) === true && hasNumber.test(str) === false) {
|
||||
// skip Dates and stuff
|
||||
if (notProper.find(tag => term.tags.has(tag))) {
|
||||
return null
|
||||
}
|
||||
// A heading can start a new clause: 'Tutorial: Creating a cake'.
|
||||
if (i === 0 && term.tags.has('Gerund') && terms[i + 1]?.tags.has('Determiner')) {
|
||||
return null
|
||||
}
|
||||
// first word in a quotation?
|
||||
if (/["']$/.test(term.pre)) {
|
||||
return null
|
||||
}
|
||||
if (term.normal === 'the') {
|
||||
return null
|
||||
}
|
||||
fillTags(terms, i, model)
|
||||
if (!term.tags.has('Noun') && !term.frozen) {
|
||||
term.tags.clear()
|
||||
}
|
||||
fastTag(term, 'ProperNoun', '2-titlecase')
|
||||
return true
|
||||
}
|
||||
//roman numberals - XVII
|
||||
if (str.length >= 2 && romanNumeral.test(str) && hasIVX.test(str) && romanNumValid.test(str) && !nope[term.normal]) {
|
||||
fastTag(term, 'RomanNumeral', '2-xvii')
|
||||
return true
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
export default checkCase
|
||||
@@ -0,0 +1,115 @@
|
||||
import colons from './1st-pass/01-colons.js'
|
||||
import hyphens from './1st-pass/02-hyphens.js'
|
||||
|
||||
import tagSwitch from './2nd-pass/00-tagSwitch.js'
|
||||
import checkCase from './2nd-pass/01-case.js'
|
||||
import checkSuffix from './2nd-pass/02-suffix.js'
|
||||
import checkRegex from './2nd-pass/03-regex.js'
|
||||
import checkPrefix from './2nd-pass/04-prefix.js'
|
||||
import checkYear from './2nd-pass/05-year.js'
|
||||
import verbType from './3rd-pass/07-verb-type.js'
|
||||
|
||||
import fillTags from './3rd-pass/_fillTags.js'
|
||||
import checkAcronym from './3rd-pass/01-acronym.js'
|
||||
import neighbours from './3rd-pass/02-neighbours.js'
|
||||
import orgWords from './3rd-pass/03-orgWords.js'
|
||||
import placeWords from './3rd-pass/04-placeWords.js'
|
||||
import nounFallback from './3rd-pass/05-fallback.js'
|
||||
import switches from './3rd-pass/06-switches.js'
|
||||
import imperative from './3rd-pass/08-imperative.js'
|
||||
|
||||
// is it all yelling-case?
|
||||
const ignoreCase = function (terms) {
|
||||
// allow 'John F Kennedy'
|
||||
if (terms.filter(t => !t.tags.has('ProperNoun')).length <= 3) {
|
||||
return false
|
||||
}
|
||||
const lowerCase = /^[a-z]/
|
||||
return terms.every(t => !lowerCase.test(t.text))
|
||||
}
|
||||
|
||||
// taggers with no clause-splitting
|
||||
const firstPass = function (docs, model, world) {
|
||||
docs.forEach(terms => {
|
||||
// check whitespace/punctuation
|
||||
colons(terms, 0, model, world)
|
||||
})
|
||||
}
|
||||
|
||||
// these methods don't care about word-neighbours
|
||||
const secondPass = function (terms, model, world, isYelling) {
|
||||
for (let i = 0; i < terms.length; i += 1) {
|
||||
// skip frozen terms, for now
|
||||
if (terms[i].frozen === true) {
|
||||
continue
|
||||
}
|
||||
// mark Noun|Verb on term metadata
|
||||
tagSwitch(terms, i, model)
|
||||
// is it titlecased?
|
||||
if (isYelling === false) {
|
||||
checkCase(terms, i, model)
|
||||
}
|
||||
// look at word ending
|
||||
checkSuffix(terms, i, model)
|
||||
// try look-like rules
|
||||
checkRegex(terms, i, model, world)
|
||||
// check for recognized prefix, like 'micro-'
|
||||
checkPrefix(terms, i, model)
|
||||
// turn '1993' into a year
|
||||
checkYear(terms, i, model)
|
||||
}
|
||||
}
|
||||
|
||||
// neighbour-based tagging
|
||||
const thirdPass = function (terms, model, world, isYelling) {
|
||||
for (let i = 0; i < terms.length; i += 1) {
|
||||
// let these tags get layered
|
||||
let found = checkAcronym(terms, i, model)
|
||||
// deduce parent tags
|
||||
fillTags(terms, i, model)
|
||||
// look left+right for hints
|
||||
found ||= neighbours(terms, i, model)
|
||||
// ¯\_(ツ)_/¯ - found nothing
|
||||
found ||= nounFallback(terms, i, model)
|
||||
}
|
||||
for (let i = 0; i < terms.length; i += 1) {
|
||||
// skip these
|
||||
if (terms[i].frozen === true) {
|
||||
continue
|
||||
}
|
||||
// Johnson LLC
|
||||
orgWords(terms, i, world, isYelling)
|
||||
// Wawel Castle
|
||||
placeWords(terms, i, world, isYelling)
|
||||
// verb-noun disambiguation, etc
|
||||
switches(terms, i, world)
|
||||
// give bare verbs more tags
|
||||
verbType(terms, i, model, world)
|
||||
// hard-nosed
|
||||
hyphens(terms, i, model, world)
|
||||
}
|
||||
// place tea bags
|
||||
imperative(terms, world)
|
||||
}
|
||||
|
||||
const preTagger = function (view) {
|
||||
const { methods, model, world } = view
|
||||
const docs = view.docs
|
||||
// try some early stuff
|
||||
firstPass(docs, model, world)
|
||||
// roughly split sentences up by clause
|
||||
const document = methods.two.quickSplit(docs)
|
||||
// start with all terms
|
||||
for (let n = 0; n < document.length; n += 1) {
|
||||
const terms = document[n]
|
||||
// is it all upper-case?
|
||||
const isYelling = ignoreCase(terms)
|
||||
// guess by the letters
|
||||
secondPass(terms, model, world, isYelling)
|
||||
// guess by the neighbours
|
||||
thirdPass(terms, model, world, isYelling)
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
export default preTagger
|
||||
@@ -0,0 +1,121 @@
|
||||
import test from 'tape'
|
||||
import nlp from '../../_lib.js'
|
||||
const here = '[two/gerund] '
|
||||
|
||||
const arr = [
|
||||
// === gerunds ===
|
||||
[`chillin'`, '#Gerund'],
|
||||
['singing', '#Gerund'],
|
||||
[`shocking quickly`, '#Gerund .'],
|
||||
[`shocking me`, '#Gerund .'],
|
||||
[`shocking his`, '#Gerund .'],
|
||||
[`he imagines shocking`, '#Pronoun #PresentTense #Gerund'],
|
||||
[`quickly shocking`, '. #Gerund'],
|
||||
[`quietly shocking`, '. #Gerund'],
|
||||
[`shocking him`, '#Gerund .'],
|
||||
[`trusting the`, '#Gerund .'],
|
||||
[`charming every`, '#Gerund .'],
|
||||
[`revealing a clue`, '#Gerund . .'],
|
||||
// [`selling like hot cakes`, '#Gerund like #Noun #Plural'],
|
||||
[`doing better for fights`, '#Gerund #Adjective for #Plural'],
|
||||
[`adding a sleeping aid.`, `#Gerund #Determiner #Noun #Singular`],
|
||||
[`babysitting her`, `#Gerund .`],
|
||||
[`baking a cooking`, `#Gerund #Determiner #Noun`],
|
||||
[`casting a shadow`, `#Gerund #Determiner #Noun`],
|
||||
[`riding that bike`, `#Gerund #Determiner #Noun`],
|
||||
[`timing the race`, `#Gerund the #Noun`],
|
||||
[`barely manufacturing`, `#Adverb #Gerund`],
|
||||
[`started thinking`, `#PastTense #Gerund`],
|
||||
[`finish listening`, `#Infinitive #Gerund`],
|
||||
[`laughing and crying`, `#Gerund and #Gerund`],
|
||||
[`without laughing`, `. #Gerund`],
|
||||
[`uplifting thinking`, `#Adjective #Noun`],
|
||||
[`he loves thinking`, `. #PresentTense #Gerund`],
|
||||
[`he stopped thinking`, `. #PastTense #Gerund`],
|
||||
[`starts thinking`, `#Verb #Gerund`],
|
||||
[`that is all you are getting`, 'that #Copula #Noun you are #Gerund'],
|
||||
// [`the rising cost of health`, 'the #Gerund #Singular of .'],
|
||||
[`shocking spencer's`, '#Gerund .'],
|
||||
[`commanding the field`, '#Gerund #Determiner #Noun'],
|
||||
['he is walking', '#Pronoun #Copula #Gerund'],
|
||||
[`revealing his guts`, '#Gerund #Possessive #Plural'],
|
||||
[`distressing us`, '#Gerund #Noun'],
|
||||
[`loving you`, '#Gerund #Noun'],
|
||||
[`promising to leave`, '#Gerund #Conjunction #Verb'],
|
||||
|
||||
// === adjectives ===
|
||||
[`as disgusting as`, '#Preposition #Adjective #Preposition'],
|
||||
[`more disgusting than`, '#Adverb #Adjective #Preposition'],
|
||||
[`was so nausiating`, '#Copula #Adverb #Adjective'],
|
||||
[`extremely moving`, '#Adverb #Adjective'],
|
||||
// [`this reckoning`, '#Determiner #Noun'],
|
||||
// [`he is redefining his art`, '#Pronoun #Copula #Gerund his #Noun'],
|
||||
// [`he is redefining art`, '#Pronoun #Copula #Gerund #Noun'],
|
||||
[`it was redefining`, '#Pronoun #Copula #Adjective'],
|
||||
[`it was a redefining moment`, '#Pronoun #Copula #Determiner #Adjective #Noun'],
|
||||
[`i found it isolating`, '#Noun #PastTense #Noun #Adjective'],
|
||||
[`it was disgusting`, '#Pronoun #Copula #Adjective'],
|
||||
[`dark green`, '#Adverb #Adjective'],
|
||||
[`kinda sparkly`, '#Adverb #Adjective'],
|
||||
[`quite stunning`, '#Adverb #Adjective'],
|
||||
[`slowly stunning`, '#Adverb #Verb'],
|
||||
[`quite awfully dashing`, '#Adverb #Adverb #Adjective'],
|
||||
['he is well', '#Pronoun #Copula #Adjective'],
|
||||
['is well made', '#Copula #Adverb #Adjective'],
|
||||
// ['tacos were way over cooked', '#Plural #Verb #Adverb #Adjective #Adjective'],
|
||||
['they were under appreciated', '#Pronoun #Verb #Adjective #Adjective'],
|
||||
['they felt appreciated', '#Pronoun #PastTense #Adjective'],
|
||||
[`a revealing clue`, 'a #Adjective .'],
|
||||
[`shocking ignorance`, '#Adjective .'],
|
||||
[`extremely charming`, '. #Adjective'],
|
||||
[`is shocking`, '. #Adjective'],
|
||||
[`really shocking`, '. #Adjective'],
|
||||
[`too insulting`, '. #Adjective'],
|
||||
[`bruising defence`, '#Adjective .'],
|
||||
[`enduring legacy`, '#Adjective .'],
|
||||
[`amazing appeal`, '#Adjective #Singular'],
|
||||
[`amazing appeals`, '#Adjective #Plural'],
|
||||
[`david is amazing`, '#Noun is #Adjective'],
|
||||
[`The difference is astounding.`, 'the #Noun is #Adjective'],
|
||||
[`their accelerating returns`, `#Noun #Adjective #Noun`],
|
||||
[`accelerating his returns`, `#Gerund his #Plural`],
|
||||
[`the exploding returns`, `. #Adjective #Plural`],
|
||||
[`their exploding returns`, `. #Adjective #Plural`],
|
||||
[`their plunging neckline`, `. #Adjective #Singular`],
|
||||
|
||||
// gerund noun
|
||||
// ['clothing', '#Noun'],
|
||||
// [`he teaches breadmaking`, 'he #Verb #Noun'],
|
||||
// [`babysitting sucks`, `#Noun #Verb`],
|
||||
// [`casting was really awful`, `#Noun #Copula #Adverb #Adjective`],
|
||||
// [`baking was fun`, `#Noun #Copula #Adjective`],
|
||||
// [`manufacturing returned`, `#Noun #PastTense`],
|
||||
// [`laughing may help`, `#Noun #Modal #Infinitive`],
|
||||
// [`laughing is pretty handy`, `#Noun #Copula #Adverb #Adjective`],
|
||||
// [`my riding`, `#Possessive #Noun`],
|
||||
// [`minus laughing`, `. #Noun`],
|
||||
// [`thinking aid`, `#Noun #Noun`],
|
||||
// [`talking therapy`, `#Noun #Noun`],
|
||||
// [`talking therapies`, `#Noun #Plural`],
|
||||
// [`toronto riding`, `#Noun #Noun`],
|
||||
// [`the timing`, `the #Noun`],
|
||||
// [`watching the working class`, `#Gerund the #Noun #Noun`],
|
||||
// [`talking the talk`, `#Gerund the #Noun`],
|
||||
// [`early warning`, `#Adjective #Noun`],
|
||||
// [`planning processes`, `#Noun #Plural`],
|
||||
// [`planning committees`, `#Noun #Plural`],
|
||||
// [`walking should be fun`, `#Noun #Modal be #Adjective`],
|
||||
// [`i think tipping sucks`, `#Pronoun #Infinitive #Noun #PresentTense`],
|
||||
// [`defeating his longstanding rivals`, '#Gerund #Possessive #Adjective #Plural'],
|
||||
]
|
||||
test('match:', function (t) {
|
||||
arr.forEach(function (a) {
|
||||
const [str, match] = a
|
||||
const doc = nlp(str).compute('tagRank')
|
||||
const tags = doc.json()[0].terms.map(term => term.tagRank[0])
|
||||
const m = doc.match(match)
|
||||
const msg = `'${(str + "' ").padEnd(20, ' ')} - '${tags.join(', ')}'`
|
||||
t.equal(m.text(), doc.text(), here + msg)
|
||||
})
|
||||
t.end()
|
||||
})
|
||||
Reference in new issue
Block a user