Upstream spencermountain/compromise state of the touched files before 435f27a18b

plugins/speech/src/compute/syllables/postProcess.js

@@ -0,0 +1,107 @@
+const starts_with_single_vowel_combos = /^(eu)/i
+const joining_consonant_vowel = /^[^aeiou]e([^d]|$)/
+const cvcv_same_consonant = /^([^aeiouy])[aeiouy]\1[aeiouy]/
+const cvcv_same_vowel = /^[^aeiouy]([aeiouy])[^aeiouy]\1/
+const cvcv_known_consonants = /^([tg][aeiouy]){2}/
+const only_one_or_more_c = /^[^aeiouy]+$/
+
+const ends_with_vowel = /[aeiouy]$/
+const starts_with_consonant_vowel = /^[^aeiouy]h?[aeiouy]/
+
+const ones = [
+  /^[^aeiou]?ion/,
+  /^[^aeiou]?ised/,
+  /^[^aeiou]?iled/,
+
+  // -ing, -ent
+  /[aeiou]n[gt]$/,
+
+  // -ate, -age
+  /\wa[gt]e$/,
+]
+
+//suffix fixes
+const postprocess = function (arr) {
+  //trim whitespace
+  arr = arr.map(function (w) {
+    return w.trim()
+  })
+  arr = arr.filter(function (w) {
+    return w !== ''
+  })
+  // if (arr.length > 2) {
+  //   return arr;
+  // }
+  const l = arr.length
+  if (l > 1) {
+    const suffix = arr[l - 2] + arr[l - 1]
+    for (let i = 0; i < ones.length; i++) {
+      if (suffix.match(ones[i])) {
+        arr[l - 2] = arr[l - 2] + arr[l - 1]
+        arr.pop()
+      }
+    }
+  }
+
+  // since the open syllable detection is overzealous,
+  // sometimes need to rejoin incorrect splits
+  if (arr.length > 1) {
+    const first_is_open =
+      (arr[0].length === 1 || arr[0].match(starts_with_consonant_vowel)) &&
+      arr[0].match(ends_with_vowel)
+    const second_is_joining = arr[1].match(joining_consonant_vowel)
+
+    if (first_is_open && second_is_joining) {
+      const possible_combination = arr[0] + arr[1]
+      const probably_separate_syllables =
+        possible_combination.match(cvcv_same_consonant) ||
+        possible_combination.match(cvcv_same_vowel) ||
+        possible_combination.match(cvcv_known_consonants)
+
+      if (!probably_separate_syllables) {
+        arr[0] = arr[0] + arr[1]
+        arr.splice(1, 1)
+      }
+    }
+  }
+
+  if (arr.length > 1) {
+    const second_to_last_is_open =
+      arr[arr.length - 2].match(starts_with_consonant_vowel) &&
+      arr[arr.length - 2].match(ends_with_vowel)
+    const last_is_joining =
+      arr[arr.length - 1].match(joining_consonant_vowel) &&
+      ones.every(re => !arr[arr.length - 1].match(re))
+
+    if (second_to_last_is_open && last_is_joining) {
+      const possible_combination = arr[arr.length - 2] + arr[arr.length - 1]
+      const probably_separate_syllables =
+        possible_combination.match(cvcv_same_consonant) ||
+        possible_combination.match(cvcv_same_vowel) ||
+        possible_combination.match(cvcv_known_consonants)
+
+      if (!probably_separate_syllables) {
+        arr[arr.length - 2] = arr[arr.length - 2] + arr[arr.length - 1]
+        arr.splice(arr.length - 1, 1)
+      }
+    }
+  }
+
+  if (arr.length > 1) {
+    const single = arr[0] + arr[1]
+    if (starts_with_single_vowel_combos.test(single)) {
+      arr[0] = single
+      arr.splice(1, 1)
+    }
+  }
+
+  if (arr.length > 1) {
+    if (only_one_or_more_c.test(arr[arr.length - 1])) {
+      arr[arr.length - 2] = arr[arr.length - 2] + arr[arr.length - 1]
+      arr.splice(arr.length - 1, 1)
+    }
+  }
+
+  return arr
+}
+export default postprocess

plugins/speech/tests/syllables.test.js

@@ -0,0 +1,73 @@
+import test from 'tape'
+import nlp from './_lib.js'
+
+const words = [
+  'sud den ly',
+  'con sti pa tion',
+  'di a bo lic',
+  'fa ted',
+  'ge ne tic',
+  'hy giene',
+  'o ma ha',
+  'i mi ta ted',
+  'tree',
+  'ci vi lised',
+  'went',
+  'to ge ther',
+  'tog gle',
+  'move ment',
+  'mo ment',
+  'do nate',
+  'vi king',
+  'wat tage',
+  'con gre gate',
+  'some thing',
+  'sales man',
+  're sour ces',
+  'eve ry thing',
+  'eve ry bo dy',
+  'hole',
+  'ho ly',
+  'sec ret',
+  'cause',
+  'fate',
+  'fates',
+  'eu lo gy',
+  'e on',
+  'rea sons',
+  'and',
+  'hoc key',
+  'wild',
+  'fun',
+  'fun ny',
+  'hor ses',
+  'cal ga ry',
+  'tor na do',
+  'pen',
+  'stu dy',
+  'ja mai ca',
+  'tri ni dad',
+  'ca na da',
+  'i ran',
+  'i raq',
+  'sau di a ra bi a',
+  'clin ton',
+  'hub ris',
+  'tur key',
+  'cor si ca',
+  'dev iled',
+  // 'horse',
+  // 'chance',
+  // 'peo ple'
+]
+
+test('test length', function (t) {
+  words.forEach(sep => {
+    const str = sep.replace(/ /g, '')
+    const doc = nlp(str)
+    const have = doc.syllables()[0]
+    const want = sep.split(/ /g)
+    t.equal(have.length, want.length, str)
+  })
+  t.end()
+})