Prd/components/backend/lib/helpdesk/glossary.rb

# frozen_string_literal: true

# Glossary - repairs domain words the transcriber cannot possibly get right.
#
# Whisper has never seen "yamt" and the hosted service accepts no vocabulary hint (it takes exactly five
# form fields, none of them a prompt), so it writes what it hears: "jamt", which is the same sound in
# Czech. No amount of audio quality fixes that, because nothing in the signal says how the word is spelt.
# A short list of terms we know are real does fix it, and does so deterministically.
#
# WHAT THIS IS NOT FOR. Only distinctive terms belong here - system names, site names, operators, product
# codes. NOT ordinary words. "balíček" is one edit from "malíček" (little finger), so listing it would
# eventually rewrite somebody's correct sentence into a wrong one. Common-word errors are a symptom of
# poor audio and are fixed at the recorder, not here. The loader enforces a minimum length and the
# matcher refuses anything but a near-miss, but the real safeguard is the list staying curated.
#
# Corrections are RETURNED as well as applied. A transcript is evidence of what a caller was told, so
# quietly editing it is not acceptable; the pipeline logs every substitution it makes.
#
# grep anchors: glossary-load, glossary-correct, glossary-match.

module Helpdesk
  module Glossary
    DEFAULT_PATH = File.join(__dir__, "glossary_terms.txt")
    MIN_TERM_LEN = 4      # shorter terms are too easy to collide with ordinary words
    # A near-miss only. One edit for a normal term, two once the word is long enough that two edits
    # still leave it unmistakable. Anything looser starts "correcting" words that were already right.
    def self.max_distance(term_len) = term_len >= 8 ? 2 : 1

    module_function

    # One term per line; "#" comments and blank lines ignored. A term may contain spaces
    # ("RF plánovač") and is matched as a phrase. grep anchor: glossary-load.
    def load(path = ENV["HELPDESK_GLOSSARY_PATH"] || DEFAULT_PATH)
      return [] unless path && File.readable?(path)
      # Explicit UTF-8, not the ambient locale. The terms are Czech, so under a non-UTF-8 LANG Ruby reads
      # them as US-ASCII and the first match raises - which the rescue below turns into an EMPTY glossary,
      # silently losing every term. The systemd unit does set LC_ALL, but anything run outside it (a cron
      # job, a one-off script, the CLI) would not, and the failure looks like "the glossary just stopped".
      File.readlines(path, chomp: true, encoding: "UTF-8")
          .map { |l| l.sub(/#.*/, "").strip }
          .reject { |l| l.empty? || l.gsub(/\s+/, "").length < MIN_TERM_LEN }
          .uniq
    rescue StandardError => e
      warn "[helpdesk] glossary load failed (#{e.class}: #{e.message}) - continuing without it"
      []
    end

    # -> [corrected_text, [{ from:, to: }, ...]]
    # Longest terms first so "RF plánovač" is considered before "plánovač" and a phrase match wins.
    # grep anchor: glossary-correct.
    def correct(text, terms = load)
      return [text, []] if text.to_s.empty? || terms.empty?
      ordered = terms.sort_by { |t| -t.split(/\s+/).length }
      changes = []
      out = text.to_s.lines.map { |line| correct_line(line, ordered, changes) }.join
      [out, changes]
    end

    # The transcript arrives as "[00:00:01] [Speaker 1]: words". Only the words may be touched - rewriting
    # a timestamp or a speaker tag would corrupt the structure everything downstream parses.
    def correct_line(line, ordered, changes)
      m = line.match(/\A(\s*(?:\[[^\]]*\]\s*)*:?\s*)(.*)\z/m)
      prefix = m ? m[1] : ""
      body   = m ? m[2] : line
      ordered.each { |term| body = apply_term(body, term, changes) }
      prefix + body
    end

    # Slide a window the width of the term across the line and replace near-misses. Case and diacritics
    # are ignored when comparing (the model often drops an accent) but the term's own spelling is what
    # gets written back. grep anchor: glossary-match.
    def apply_term(body, term, changes)
      tw = term.split(/\s+/)
      n = tw.length
      tokens = body.split(/(\s+)/)                        # keep the whitespace so spacing survives
      words = tokens.each_index.select { |i| i.even? }    # even indices are words
      return body if words.length < n

      i = 0
      while i <= words.length - n
        idx = words[i, n]
        phrase = idx.map { |k| tokens[k] }
        core = phrase.map { |w| strip_edges(w) }
        if n == 1 && core.first && (split = fused_split(core.first, term))
          # A term swallowed by the word before it. Seen on a real call: "přes yamt" came back as
          # "přezjamt" - fused AND assimilated (přes -> přez before a voiced consonant), so the token
          # never resembles the term on its own. Restore both halves.
          changes << { from: core.first, to: "#{split[0]} #{term}" }
          lead, trail = edges(phrase.first, phrase.last)
          tokens[idx.first] = lead + split[0] + " " + term + trail
          i += 1
        elsif core.all? { |c| !c.empty? } && near_miss?(core.join(" "), term)
          original = core.join(" ")
          unless original == term
            changes << { from: original, to: term }
            lead, trail = edges(phrase.first, phrase.last)
            # Collapse the whole phrase into its first token: blank everything from just after it up to
            # and including the last word, which clears the separators between them too. Doing this by
            # word index alone would leave the spaces behind and pad the line.
            ((idx.first + 1)..idx.last).each { |k| tokens[k] = "" }
            tokens[idx.first] = lead + term + trail
          end
          i += n
        else
          i += 1
        end
      end
      tokens.join
    end

    # Punctuation clings to spoken words ("jamt," / "(jamt)"); compare the word itself, restore the rest.
    def strip_edges(w) = w.gsub(/\A[^[:alnum:]]+|[^[:alnum:]]+\z/, "")

    def edges(first, last)
      [first[/\A[^[:alnum:]]*/].to_s, last[/[^[:alnum:]]*\z/].to_s]
    end

    # Czech prepositions that can swallow the following word in fast speech. Listed explicitly rather
    # than accepting any short prefix, because "any prefix" would let the glossary hack a term out of
    # the middle of ordinary words. Includes the voiced-assimilated forms the transcriber actually
    # writes ("přes" heard as "přez", "s" as "z"), since that is what the audio sounds like.
    FUSING_PREFIXES = %w[pres přes prez přez na do od pod nad pro pri při bez pres pri za pres
                         v ve z ze s se k ke o u].freeze

    # Did a preposition swallow the term? -> [prefix, term] or nil. grep anchor: glossary-fused.
    def fused_split(token, term)
      t = fold(token)
      b = fold(term)
      return nil if t.length <= b.length || t.length - b.length > 5
      # The tail must be the term (a near-miss is fine: "jamt" for "yamt"), and what precedes it must be
      # a real preposition, not just any letters.
      tail = t[-b.length..]
      return nil unless tail == b || levenshtein(tail, b) <= 1
      head = t[0...(t.length - b.length)]
      return nil unless FUSING_PREFIXES.include?(head)
      [token[0...(token.length - b.length)], term]
    end

    def near_miss?(candidate, term)
      return false if candidate.to_s.empty? || term.to_s.empty?
      return false if candidate == term                   # byte-identical, nothing to do
      # Capitalisation is not a misrecognition. Rewriting "Repeater" at the start of a sentence to the
      # lower-case listed form damages the text, and forcing a proper noun's case is not this tool's job
      # either. Spelling only. Accents still get restored below, because "Jezlovice" and "Ježlovice"
      # differ by more than case.
      return false if candidate.downcase == term.downcase
      a = fold(candidate)
      b = fold(term)
      return false if (a.length - b.length).abs > 2       # wildly different length is a different word
      # Folded-equal means the ONLY difference was accents ("Jezlovice" for "Ježlovice"). Always worth
      # restoring, and it has to be decided before the inflection guard below, which would otherwise
      # see two identical strings as "differing only in the ending" and refuse.
      return true if a == b
      return false if suffix_only_difference?(a, b)
      levenshtein(a, b) <= max_distance(b.length)
    end

    # Czech inflects heavily: "na Vodafonu" is the correct locative of "Vodafone", and "yamtu" is a
    # perfectly good genitive. Those differ from the listed form only at the end, so a difference confined
    # to the last couple of characters is grammar, not a misrecognition, and rewriting it would replace
    # correct Czech with a wrong case. Misrecognitions of the kind this exists for ("jamt" for "yamt")
    # differ at the start or in the middle instead.
    def suffix_only_difference?(a, b)
      n = [a.length, b.length].min
      shared = 0
      shared += 1 while shared < n && a[shared] == b[shared]
      shared >= 3 && shared >= n - 2
    end

    def fold(s)
      s.downcase.tr("áčďéěíňóřšťúůýž", "acdeeinorstuuyz")
    end

    def levenshtein(a, b)
      ac = a.chars; bc = b.chars
      prev = (0..bc.length).to_a
      ac.each_with_index do |x, i|
        cur = [i + 1]
        bc.each_with_index { |y, j| cur << [prev[j + 1] + 1, cur[j] + 1, prev[j] + (x == y ? 0 : 1)].min }
        prev = cur
      end
      prev[bc.length]
    end
  end
end