Upstream spencermountain/compromise state of the touched files before 8787708938

src/2-two/preTagger/compute/tagger/2nd-pass/01-case.js

@@ -0,0 +1,61 @@
+import fastTag from '../_fastTag.js'
+import fillTags from '../3rd-pass/_fillTags.js'
+
+const titleCase = /^\p{Lu}[\p{Ll}'’]/u
+const hasNumber = /[0-9]/
+const notProper = ['Date', 'Month', 'WeekDay', 'Unit', 'Expression']
+
+// roman numeral by regex
+const hasIVX = /[IVX]/ // does it ~look like~ a roman numeral?
+// quick-version
+const romanNumeral = /^[IVXLCDM]{2,}$/
+// https://stackoverflow.com/a/267405/168877
+const romanNumValid = /^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$/
+const nope = {
+  li: true,
+  dc: true,
+  md: true,
+  dm: true,
+  ml: true,
+}
+
+// if it's a unknown titlecase word, it's a propernoun
+const checkCase = function (terms, i, model) {
+  const term = terms[i]
+  // assume terms are already indexed
+  term.index ||= [0, 0]
+  const index = term.index[1]
+  const str = term.text || '' //need case info
+  // titlecase and not first word of sentence
+  if (index !== 0 && titleCase.test(str) === true && hasNumber.test(str) === false) {
+    // skip Dates and stuff
+    if (notProper.find(tag => term.tags.has(tag))) {
+      return null
+    }
+    // A heading can start a new clause: 'Tutorial: Creating a cake'.
+    if (i === 0 && term.tags.has('Gerund') && terms[i + 1]?.tags.has('Determiner')) {
+      return null
+    }
+    // first word in a quotation?
+    if (/["']$/.test(term.pre)) {
+      return null
+    }
+    if (term.normal === 'the') {
+      return null
+    }
+    fillTags(terms, i, model)
+    if (!term.tags.has('Noun') && !term.frozen) {
+      term.tags.clear()
+    }
+    fastTag(term, 'ProperNoun', '2-titlecase')
+    return true
+  }
+  //roman numberals - XVII
+  if (str.length >= 2 && romanNumeral.test(str) && hasIVX.test(str) && romanNumValid.test(str) && !nope[term.normal]) {
+    fastTag(term, 'RomanNumeral', '2-xvii')
+    return true
+  }
+
+  return null
+}
+export default checkCase

src/2-two/preTagger/compute/tagger/index.js

@@ -0,0 +1,115 @@
+import colons from './1st-pass/01-colons.js'
+import hyphens from './1st-pass/02-hyphens.js'
+
+import tagSwitch from './2nd-pass/00-tagSwitch.js'
+import checkCase from './2nd-pass/01-case.js'
+import checkSuffix from './2nd-pass/02-suffix.js'
+import checkRegex from './2nd-pass/03-regex.js'
+import checkPrefix from './2nd-pass/04-prefix.js'
+import checkYear from './2nd-pass/05-year.js'
+import verbType from './3rd-pass/07-verb-type.js'
+
+import fillTags from './3rd-pass/_fillTags.js'
+import checkAcronym from './3rd-pass/01-acronym.js'
+import neighbours from './3rd-pass/02-neighbours.js'
+import orgWords from './3rd-pass/03-orgWords.js'
+import placeWords from './3rd-pass/04-placeWords.js'
+import nounFallback from './3rd-pass/05-fallback.js'
+import switches from './3rd-pass/06-switches.js'
+import imperative from './3rd-pass/08-imperative.js'
+
+// is it all yelling-case?
+const ignoreCase = function (terms) {
+  // allow 'John F Kennedy'
+  if (terms.filter(t => !t.tags.has('ProperNoun')).length <= 3) {
+    return false
+  }
+  const lowerCase = /^[a-z]/
+  return terms.every(t => !lowerCase.test(t.text))
+}
+
+// taggers with no clause-splitting
+const firstPass = function (docs, model, world) {
+  docs.forEach(terms => {
+    // check whitespace/punctuation
+    colons(terms, 0, model, world)
+  })
+}
+
+// these methods don't care about word-neighbours
+const secondPass = function (terms, model, world, isYelling) {
+  for (let i = 0; i < terms.length; i += 1) {
+    // skip frozen terms, for now
+    if (terms[i].frozen === true) {
+      continue
+    }
+    // mark Noun|Verb on term metadata
+    tagSwitch(terms, i, model)
+    //  is it titlecased?
+    if (isYelling === false) {
+      checkCase(terms, i, model)
+    }
+    // look at word ending
+    checkSuffix(terms, i, model)
+    // try look-like rules
+    checkRegex(terms, i, model, world)
+    // check for recognized prefix, like 'micro-'
+    checkPrefix(terms, i, model)
+    // turn '1993' into a year
+    checkYear(terms, i, model)
+  }
+}
+
+// neighbour-based tagging
+const thirdPass = function (terms, model, world, isYelling) {
+  for (let i = 0; i < terms.length; i += 1) {
+    // let these tags get layered
+    let found = checkAcronym(terms, i, model)
+    // deduce parent tags
+    fillTags(terms, i, model)
+    // look left+right for hints
+    found ||= neighbours(terms, i, model)
+    //  ¯\_(ツ)_/¯ - found nothing
+    found ||= nounFallback(terms, i, model)
+  }
+  for (let i = 0; i < terms.length; i += 1) {
+    // skip these
+    if (terms[i].frozen === true) {
+      continue
+    }
+    // Johnson LLC
+    orgWords(terms, i, world, isYelling)
+    // Wawel Castle
+    placeWords(terms, i, world, isYelling)
+    // verb-noun disambiguation, etc
+    switches(terms, i, world)
+    // give bare verbs more tags
+    verbType(terms, i, model, world)
+    // hard-nosed
+    hyphens(terms, i, model, world)
+  }
+  // place tea bags
+  imperative(terms, world)
+}
+
+const preTagger = function (view) {
+  const { methods, model, world } = view
+  const docs = view.docs
+  // try some early stuff
+  firstPass(docs, model, world)
+  // roughly split sentences up by clause
+  const document = methods.two.quickSplit(docs)
+  // start with all terms
+  for (let n = 0; n < document.length; n += 1) {
+    const terms = document[n]
+    // is it all upper-case?
+    const isYelling = ignoreCase(terms)
+    // guess by the letters
+    secondPass(terms, model, world, isYelling)
+    // guess by the neighbours
+    thirdPass(terms, model, world, isYelling)
+  }
+  return document
+}
+
+export default preTagger

tests/two/tagger/ambiguity/gerund.test.js

@@ -0,0 +1,121 @@
+import test from 'tape'
+import nlp from '../../_lib.js'
+const here = '[two/gerund] '
+
+const arr = [
+  // === gerunds ===
+  [`chillin'`, '#Gerund'],
+  ['singing', '#Gerund'],
+  [`shocking quickly`, '#Gerund .'],
+  [`shocking me`, '#Gerund .'],
+  [`shocking his`, '#Gerund .'],
+  [`he imagines shocking`, '#Pronoun #PresentTense #Gerund'],
+  [`quickly shocking`, '. #Gerund'],
+  [`quietly shocking`, '. #Gerund'],
+  [`shocking him`, '#Gerund .'],
+  [`trusting the`, '#Gerund .'],
+  [`charming every`, '#Gerund .'],
+  [`revealing a clue`, '#Gerund . .'],
+  // [`selling like hot cakes`, '#Gerund like #Noun #Plural'],
+  [`doing better for fights`, '#Gerund #Adjective for #Plural'],
+  [`adding a sleeping aid.`, `#Gerund #Determiner #Noun #Singular`],
+  [`babysitting her`, `#Gerund .`],
+  [`baking a cooking`, `#Gerund #Determiner #Noun`],
+  [`casting a shadow`, `#Gerund #Determiner #Noun`],
+  [`riding that bike`, `#Gerund #Determiner #Noun`],
+  [`timing the race`, `#Gerund the #Noun`],
+  [`barely manufacturing`, `#Adverb #Gerund`],
+  [`started thinking`, `#PastTense #Gerund`],
+  [`finish listening`, `#Infinitive #Gerund`],
+  [`laughing and crying`, `#Gerund and #Gerund`],
+  [`without laughing`, `. #Gerund`],
+  [`uplifting thinking`, `#Adjective #Noun`],
+  [`he loves thinking`, `. #PresentTense #Gerund`],
+  [`he stopped thinking`, `. #PastTense #Gerund`],
+  [`starts thinking`, `#Verb #Gerund`],
+  [`that is all you are getting`, 'that #Copula #Noun you are #Gerund'],
+  // [`the rising cost of health`, 'the #Gerund #Singular of .'],
+  [`shocking spencer's`, '#Gerund .'],
+  [`commanding the field`, '#Gerund #Determiner #Noun'],
+  ['he is walking', '#Pronoun #Copula #Gerund'],
+  [`revealing his guts`, '#Gerund #Possessive #Plural'],
+  [`distressing us`, '#Gerund #Noun'],
+  [`loving you`, '#Gerund #Noun'],
+  [`promising to leave`, '#Gerund #Conjunction #Verb'],
+
+  // === adjectives ===
+  [`as disgusting as`, '#Preposition #Adjective #Preposition'],
+  [`more disgusting than`, '#Adverb #Adjective #Preposition'],
+  [`was so nausiating`, '#Copula #Adverb #Adjective'],
+  [`extremely moving`, '#Adverb #Adjective'],
+  // [`this reckoning`, '#Determiner #Noun'],
+  // [`he is redefining his art`, '#Pronoun #Copula #Gerund his #Noun'],
+  // [`he is redefining art`, '#Pronoun #Copula #Gerund #Noun'],
+  [`it was redefining`, '#Pronoun #Copula #Adjective'],
+  [`it was a redefining moment`, '#Pronoun #Copula #Determiner #Adjective #Noun'],
+  [`i found it isolating`, '#Noun #PastTense #Noun #Adjective'],
+  [`it was disgusting`, '#Pronoun #Copula #Adjective'],
+  [`dark green`, '#Adverb #Adjective'],
+  [`kinda sparkly`, '#Adverb #Adjective'],
+  [`quite stunning`, '#Adverb #Adjective'],
+  [`slowly stunning`, '#Adverb #Verb'],
+  [`quite awfully dashing`, '#Adverb #Adverb #Adjective'],
+  ['he is well', '#Pronoun #Copula #Adjective'],
+  ['is well made', '#Copula #Adverb #Adjective'],
+  // ['tacos were way over cooked', '#Plural #Verb #Adverb #Adjective #Adjective'],
+  ['they were under appreciated', '#Pronoun #Verb #Adjective #Adjective'],
+  ['they felt appreciated', '#Pronoun #PastTense #Adjective'],
+  [`a revealing clue`, 'a #Adjective .'],
+  [`shocking ignorance`, '#Adjective .'],
+  [`extremely charming`, '. #Adjective'],
+  [`is shocking`, '. #Adjective'],
+  [`really shocking`, '. #Adjective'],
+  [`too insulting`, '. #Adjective'],
+  [`bruising defence`, '#Adjective .'],
+  [`enduring legacy`, '#Adjective .'],
+  [`amazing appeal`, '#Adjective #Singular'],
+  [`amazing appeals`, '#Adjective #Plural'],
+  [`david is amazing`, '#Noun is #Adjective'],
+  [`The difference is astounding.`, 'the #Noun is #Adjective'],
+  [`their accelerating returns`, `#Noun #Adjective #Noun`],
+  [`accelerating his returns`, `#Gerund his #Plural`],
+  [`the exploding returns`, `. #Adjective #Plural`],
+  [`their exploding returns`, `. #Adjective #Plural`],
+  [`their plunging neckline`, `. #Adjective #Singular`],
+
+  // gerund noun
+  // ['clothing', '#Noun'],
+  // [`he teaches breadmaking`, 'he #Verb #Noun'],
+  // [`babysitting sucks`, `#Noun #Verb`],
+  // [`casting was really awful`, `#Noun #Copula #Adverb #Adjective`],
+  // [`baking was fun`, `#Noun #Copula #Adjective`],
+  // [`manufacturing returned`, `#Noun #PastTense`],
+  // [`laughing may help`, `#Noun #Modal #Infinitive`],
+  // [`laughing is pretty handy`, `#Noun #Copula #Adverb #Adjective`],
+  // [`my riding`, `#Possessive #Noun`],
+  // [`minus laughing`, `. #Noun`],
+  // [`thinking aid`, `#Noun #Noun`],
+  // [`talking therapy`, `#Noun #Noun`],
+  // [`talking therapies`, `#Noun #Plural`],
+  // [`toronto riding`, `#Noun #Noun`],
+  // [`the timing`, `the #Noun`],
+  // [`watching the working class`, `#Gerund the #Noun #Noun`],
+  // [`talking the talk`, `#Gerund the #Noun`],
+  // [`early warning`, `#Adjective #Noun`],
+  // [`planning processes`, `#Noun #Plural`],
+  // [`planning committees`, `#Noun #Plural`],
+  // [`walking should be fun`, `#Noun #Modal be #Adjective`],
+  // [`i think tipping sucks`, `#Pronoun #Infinitive #Noun #PresentTense`],
+  // [`defeating his longstanding rivals`, '#Gerund #Possessive #Adjective #Plural'],
+]
+test('match:', function (t) {
+  arr.forEach(function (a) {
+    const [str, match] = a
+    const doc = nlp(str).compute('tagRank')
+    const tags = doc.json()[0].terms.map(term => term.tagRank[0])
+    const m = doc.match(match)
+    const msg = `'${(str + "' ").padEnd(20, ' ')}  - '${tags.join(', ')}'`
+    t.equal(m.text(), doc.text(), here + msg)
+  })
+  t.end()
+})