Prd/components/transcription-worker/bench/test_ab.rb

# frozen_string_literal: true

# Offline tests for the A/B harness scoring (no gems, no network, no audio).
# The scoring is the part worth testing: if WER or the markup stripping is wrong, every comparison the
# harness prints is wrong too, and it would look authoritative while being useless.
# Run: ruby components/transcription-worker/bench/test_ab.rb
require_relative "ab"

$pass = 0; $fail = 0
def ok(d); $pass += 1; puts "  ok  #{d}"; end
def bad(d, got = nil); $fail += 1; puts "  FAIL #{d}#{got.nil? ? '' : "  (got: #{got.inspect})"}"; end
def eq(a, b, d); a == b ? ok(d) : bad(d, a); end
def close(a, b, d, tol = 0.001); (a - b).abs < tol ? ok(d) : bad(d, a); end
def truthy(x, d); x ? ok(d) : bad(d, x); end
def falsy(x, d); x ? bad(d, x) : ok(d); end

B = Helpdesk::Bench

puts "stripping the service's markup down to words"
real = "[00:00:01] [Speaker 1]: Ano, ano, dobré ráno.\n\n[00:00:03] [Speaker 2]: Dobrý den."
eq(B.words(real), %w[ano ano dobré ráno dobrý den], "markup stripped, punctuation dropped, words kept")

puts "normalising compares like with like"
eq(B.words("Dobrý den!"), B.words("dobrý, den."), "case and punctuation are not transcription errors")
eq(B.words("a   b\n\nc"), %w[a b c], "whitespace collapsed")
# Czech diacritics change the word, so folding them would hide real mistakes.
truthy(B.words("balíček") != B.words("balicek"), "diacritics are NOT folded (balíček != balicek)")

puts "word error rate"
close(B.wer("a b c d", "a b c d")[:wer], 0.0, "identical -> 0")
close(B.wer("a b c d", "a b c x")[:wer], 0.25, "one substitution in four -> 0.25")
close(B.wer("a b c d", "a b c")[:wer],   0.25, "one deletion -> 0.25")
close(B.wer("a b c d", "a b c d e")[:wer], 0.25, "one insertion -> 0.25")
close(B.wer("a b", "x y z w")[:wer], 2.0, "a hallucinated run can exceed 1.0 and is NOT clamped")
eq(B.wer("", "anything")[:wer], nil, "no reference -> no score, rather than a misleading zero")
eq(B.wer("a b c d", "a b c x")[:ref_words], 4, "reference length reported")

puts "the real errors this harness exists to catch"
ref = "Máme tady od vás zmeškaný hovor. Potřeboval jsem doručit balíček od DPD."
hyp = "Máme tady od vás myštěný hovor. Potřeboval jsem doručit badíček od DPD."
r = B.wer(ref, hyp)
close(r[:wer], 2.0 / 12, "two wrong words out of twelve")
d = B.word_diff(ref, hyp)
truthy(d[:missing].include?("zmeškaný") && d[:missing].include?("balíček"), "names the words that went missing")
truthy(d[:spurious].include?("myštěný") && d[:spurious].include?("badíček"), "names what was invented in their place")

puts "alignment says WHERE two transcripts diverge, not just how much"
ops = B.align(%w[a b c d], %w[a x c d])
eq(ops.map(&:first), %i[ok sub ok ok], "a substitution is located, not just counted")
eq(ops[1][1..2], %w[b x], "and it carries both words")
eq(B.align(%w[a b], %w[a]).map(&:first), %i[ok del], "a dropped word is a deletion")
eq(B.align(%w[a], %w[a b]).map(&:first), %i[ok ins], "an added word is an insertion")

puts "classifying disagreements: only genuine misrecognitions should count"
eq(B.classify_chunk(%w[sim kartou], %w[simkartou]), :boundary, "same letters, different spacing")
eq(B.classify_chunk(%w[na shledanou], %w[nashledanou]), :boundary, "a joined phrase is a boundary case")
eq(B.classify_chunk(%w[balicek], %w[balíček]), :diacritic, "diacritics only")
eq(B.classify_chunk(%w[děkuju], %w[děkuji]), :variant, "spoken -u vs written -i is not an error")
eq(B.classify_chunk(%w[potřebuju], %w[potřebuji]), :variant, "same rule, another verb")
eq(B.classify_chunk(%w[čtyřicet], %w[šedesát]), :number, "digits are their own failure mode")
eq(B.classify_chunk(%w[balíček], %w[badíček]), :real, "a genuinely misheard word counts")
eq(B.classify_chunk(%w[zmeškaný], %w[myštěný]), :real, "and so does an unrecognisable one")

puts "the spoken-variant rule is narrow on purpose"
truthy(B.spoken_variant?("děkuju", "děkuji"), "u <-> i on the same stem")
falsy(B.spoken_variant?("volal", "vál"), "different stems are NOT a variant")
falsy(B.spoken_variant?("ten", "tex"), "the ending must actually be u or i")
falsy(B.spoken_variant?("du", "di"), "too short to be confident")

puts "a mixed run of edits is decomposed, not lumped together"
cs = B.chunks("děkuju mockrát na shledanou", "děkuji moc krát nashledanou")
truthy(cs.length >= 2, "the run is split rather than lumped into one verdict")
eq(cs.map { |c| c[:kind] }.uniq.sort, %i[boundary variant], "each part gets its own explanation")
falsy(cs.any? { |c| c[:kind] == :real }, "and none of them is a misrecognition")
# Adjacent words that jointly form a pure boundary case stay together on purpose: splitting
# "mockrát na shledanou" -> "moc krát nashledanou" any further would invent distinctions.
eq(B.chunks("mockrát na shledanou", "moc krát nashledanou").length, 1, "one phenomenon stays one chunk")

puts "effective WER drops the explainable, raw WER does not"
r = "děkuju mockrát a na shledanou"
h = "děkuji moc krát a nashledanou"
truthy(B.wer(r, h)[:wer] > 0, "raw WER counts formatting as error")
eq(B.effective_wer(r, h)[:wer], 0.0, "effective WER sees nothing worth reporting")
truthy(B.effective_wer("balíček je tady", "badíček je tady")[:wer] > 0, "a real error still counts")

puts "character error rate ignores where word boundaries fell"
eq(B.cer("na shledanou", "nashledanou"), 0.0, "identical letters -> 0 regardless of spacing")
truthy(B.cer("balíček", "badíček") > 0, "a wrong letter still shows")

puts "per-channel scoring, because a merged transcript must not be scored linearly"
merged = "[00:00:01] [Operator]: dobrý den\n\n[00:00:05] [Caller]: mám problém\n\n[00:00:09] [Operator]: rozumím\n"
eq(B.speaker_side(merged, "Operator").split.join(" "), "dobrý den rozumím", "one speaker's words extracted")
eq(B.speaker_side(merged, "Caller").split.join(" "), "mám problém", "and the other's, separately")
eq(B.speaker_side(merged, "Nobody"), "", "an absent label yields nothing rather than everything")

sc = B.score_channels(merged, { "Operator" => "dobrý den rozumím", "Caller" => "mám problém" })
close(sc["Operator"][:wer], 0.0, "a perfect operator channel scores 0")
close(sc["Caller"][:wer], 0.0, "a perfect caller channel scores 0")
truthy(sc["Operator"].key?(:cer), "character rate reported per channel too")

# The bug this exists to avoid: scored linearly against a reading-order script, correct text looks awful
# purely because the merge is ordered by time. Per channel, it is correctly recognised as perfect.
script = "dobrý den rozumím mám problém"
truthy(B.effective_wer(script, B.strip_markup(merged)).fetch(:wer) > 0.3,
       "linear scoring of a correct merge reports a large bogus error")
close(B.score_channels(merged, { "Operator" => "dobrý den rozumím", "Caller" => "mám problém" })["Operator"][:wer],
      0.0, "per-channel scoring of the same transcript reports none")

puts "speaker counting, for judging diarisation"
two = "[00:00:01] [Speaker 1]: a\n[00:00:04] [Speaker 2]: b\n[00:00:09] [Speaker 1]: c"
eq(B.speaker_count(two), 2, "two voices counted once each")
eq(B.speaker_count(two + "\n[00:00:12] [Speaker 3]: d"), 3, "a split speaker shows up as three")
eq(B.speaker_count("[00:00:01] [Speaker 1]: only me"), 1, "merged speakers show up as one")
eq(B.speaker_count("no tags at all"), 0, "untagged text -> 0, not a crash")

puts "variants cover the questions we actually have"
names = B::VARIANTS.map { |v| v[:name] }
truthy(names.include?("baseline"), "keeps today's production settings as the control")
truthy(B::VARIANTS.any? { |v| v[:model] == "large-v3" }, "tests the model question")
truthy(B::VARIANTS.any? { |v| v[:max_speakers] == 2 }, "tests the diarisation question")
eq(names.uniq.length, names.length, "variant names are unique (they are used as cache filenames)")

puts
puts "#{$pass} passed, #{$fail} failed"
exit($fail.zero? ? 0 : 1)