Upstream spencermountain/compromise state of the touched files before 8787708938

src/2-two/preTagger/compute/tagger/2nd-pass/01-case.js

@@ -0,0 +1,61 @@
+import fastTag from '../_fastTag.js'
+import fillTags from '../3rd-pass/_fillTags.js'
+
+const titleCase = /^\p{Lu}[\p{Ll}'’]/u
+const hasNumber = /[0-9]/
+const notProper = ['Date', 'Month', 'WeekDay', 'Unit', 'Expression']
+
+// roman numeral by regex
+const hasIVX = /[IVX]/ // does it ~look like~ a roman numeral?
+// quick-version
+const romanNumeral = /^[IVXLCDM]{2,}$/
+// https://stackoverflow.com/a/267405/168877
+const romanNumValid = /^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$/
+const nope = {
+  li: true,
+  dc: true,
+  md: true,
+  dm: true,
+  ml: true,
+}
+
+// if it's a unknown titlecase word, it's a propernoun
+const checkCase = function (terms, i, model) {
+  const term = terms[i]
+  // assume terms are already indexed
+  term.index ||= [0, 0]
+  const index = term.index[1]
+  const str = term.text || '' //need case info
+  // titlecase and not first word of sentence
+  if (index !== 0 && titleCase.test(str) === true && hasNumber.test(str) === false) {
+    // skip Dates and stuff
+    if (notProper.find(tag => term.tags.has(tag))) {
+      return null
+    }
+    // A heading can start a new clause: 'Tutorial: Creating a cake'.
+    if (i === 0 && term.tags.has('Gerund') && terms[i + 1]?.tags.has('Determiner')) {
+      return null
+    }
+    // first word in a quotation?
+    if (/["']$/.test(term.pre)) {
+      return null
+    }
+    if (term.normal === 'the') {
+      return null
+    }
+    fillTags(terms, i, model)
+    if (!term.tags.has('Noun') && !term.frozen) {
+      term.tags.clear()
+    }
+    fastTag(term, 'ProperNoun', '2-titlecase')
+    return true
+  }
+  //roman numberals - XVII
+  if (str.length >= 2 && romanNumeral.test(str) && hasIVX.test(str) && romanNumValid.test(str) && !nope[term.normal]) {
+    fastTag(term, 'RomanNumeral', '2-xvii')
+    return true
+  }
+
+  return null
+}
+export default checkCase

src/2-two/preTagger/compute/tagger/index.js

@@ -0,0 +1,115 @@
+import colons from './1st-pass/01-colons.js'
+import hyphens from './1st-pass/02-hyphens.js'
+
+import tagSwitch from './2nd-pass/00-tagSwitch.js'
+import checkCase from './2nd-pass/01-case.js'
+import checkSuffix from './2nd-pass/02-suffix.js'
+import checkRegex from './2nd-pass/03-regex.js'
+import checkPrefix from './2nd-pass/04-prefix.js'
+import checkYear from './2nd-pass/05-year.js'
+import verbType from './3rd-pass/07-verb-type.js'
+
+import fillTags from './3rd-pass/_fillTags.js'
+import checkAcronym from './3rd-pass/01-acronym.js'
+import neighbours from './3rd-pass/02-neighbours.js'
+import orgWords from './3rd-pass/03-orgWords.js'
+import placeWords from './3rd-pass/04-placeWords.js'
+import nounFallback from './3rd-pass/05-fallback.js'
+import switches from './3rd-pass/06-switches.js'
+import imperative from './3rd-pass/08-imperative.js'
+
+// is it all yelling-case?
+const ignoreCase = function (terms) {
+  // allow 'John F Kennedy'
+  if (terms.filter(t => !t.tags.has('ProperNoun')).length <= 3) {
+    return false
+  }
+  const lowerCase = /^[a-z]/
+  return terms.every(t => !lowerCase.test(t.text))
+}
+
+// taggers with no clause-splitting
+const firstPass = function (docs, model, world) {
+  docs.forEach(terms => {
+    // check whitespace/punctuation
+    colons(terms, 0, model, world)
+  })
+}
+
+// these methods don't care about word-neighbours
+const secondPass = function (terms, model, world, isYelling) {
+  for (let i = 0; i < terms.length; i += 1) {
+    // skip frozen terms, for now
+    if (terms[i].frozen === true) {
+      continue
+    }
+    // mark Noun|Verb on term metadata
+    tagSwitch(terms, i, model)
+    //  is it titlecased?
+    if (isYelling === false) {
+      checkCase(terms, i, model)
+    }
+    // look at word ending
+    checkSuffix(terms, i, model)
+    // try look-like rules
+    checkRegex(terms, i, model, world)
+    // check for recognized prefix, like 'micro-'
+    checkPrefix(terms, i, model)
+    // turn '1993' into a year
+    checkYear(terms, i, model)
+  }
+}
+
+// neighbour-based tagging
+const thirdPass = function (terms, model, world, isYelling) {
+  for (let i = 0; i < terms.length; i += 1) {
+    // let these tags get layered
+    let found = checkAcronym(terms, i, model)
+    // deduce parent tags
+    fillTags(terms, i, model)
+    // look left+right for hints
+    found ||= neighbours(terms, i, model)
+    //  ¯\_(ツ)_/¯ - found nothing
+    found ||= nounFallback(terms, i, model)
+  }
+  for (let i = 0; i < terms.length; i += 1) {
+    // skip these
+    if (terms[i].frozen === true) {
+      continue
+    }
+    // Johnson LLC
+    orgWords(terms, i, world, isYelling)
+    // Wawel Castle
+    placeWords(terms, i, world, isYelling)
+    // verb-noun disambiguation, etc
+    switches(terms, i, world)
+    // give bare verbs more tags
+    verbType(terms, i, model, world)
+    // hard-nosed
+    hyphens(terms, i, model, world)
+  }
+  // place tea bags
+  imperative(terms, world)
+}
+
+const preTagger = function (view) {
+  const { methods, model, world } = view
+  const docs = view.docs
+  // try some early stuff
+  firstPass(docs, model, world)
+  // roughly split sentences up by clause
+  const document = methods.two.quickSplit(docs)
+  // start with all terms
+  for (let n = 0; n < document.length; n += 1) {
+    const terms = document[n]
+    // is it all upper-case?
+    const isYelling = ignoreCase(terms)
+    // guess by the letters
+    secondPass(terms, model, world, isYelling)
+    // guess by the neighbours
+    thirdPass(terms, model, world, isYelling)
+  }
+  return document
+}
+
+export default preTagger