more LM baseline

README.md

@@ -27,27 +27,47 @@ The leaderboard shows WER metrics for multiple speech recognition sources as col
 - Tedlium-3
 - OVERALL (aggregate across all sources)
 
-## Metrics
+## Baseline Methods
 
-The leaderboard displays as rows:
-- **Count**: Number of examples in the test set for each source
-- **No LM Baseline**: Word Error Rate between the reference transcription and 1-best ASR output without language model correction
+The leaderboard displays three baseline approaches:
 
-## Baseline Calculation
+1. **No LM Baseline**: Uses the 1-best ASR output without any correction (input1)
+2. **N-best LM Ranking**: Ranks the N-best hypotheses using a simple language model approach and chooses the best one
+3. **N-best Correction**: Uses a voting-based method to correct the transcript by combining information from all N-best hypotheses
 
-Word Error Rate is calculated between:
-- Reference transcription ("transcription" field)
-- 1-best ASR output ("input1" field or first item from "hypothesis" when input1 is unavailable)
+## Metrics
 
-Lower WER values indicate better transcription accuracy.
+The leaderboard displays as rows:
+- **Number of Examples**: Count of examples in the test set for each source
+- **Word Error Rate (No LM)**: WER between reference and 1-best ASR output
+- **Word Error Rate (N-best LM Ranking)**: WER between reference and LM-ranked best hypothesis
+- **Word Error Rate (N-best Correction)**: WER between reference and the corrected N-best hypothesis
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Lower WER values indicate better transcription accuracy.
 
 ## Table Structure
 
 The leaderboard is displayed as a table with:
 
-- **Rows**: "Number of Examples" and "Word Error Rate (WER)"
+- **Rows**: Different metrics (example counts and WER values for each method)
 - **Columns**: Different data sources (CHiME4, CORAAL, CommonVoice, etc.) and OVERALL
 
 Each cell shows the corresponding metric for that specific data source. The OVERALL column shows aggregate metrics across all sources.
+
+## Technical Details
+
+### N-best LM Ranking
+This method scores each hypothesis in the N-best list using:
+- N-gram statistics (bigrams)
+- Text length
+- N-gram variety
+
+The hypothesis with the highest score is selected.
+
+### N-best Correction
+This method uses a simple voting mechanism:
+- Groups hypotheses of the same length
+- For each word position, chooses the most common word across all hypotheses
+- Constructs a new transcript from these voted words
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -6,6 +6,8 @@ import numpy as np
 from functools import lru_cache
 import traceback
 import re
+import string
+from collections import Counter
 
 # Cache the dataset loading to avoid reloading on refresh
 @lru_cache(maxsize=1)
@@ -37,6 +39,100 @@ def preprocess_text(text):
     text = re.sub(r'\s+', ' ', text).strip()
     return text
 
+# Simple language model scoring - count n-grams
+def score_hypothesis(hypothesis, n=4):
+    """Score a hypothesis using simple n-gram statistics"""
+    if not hypothesis:
+        return 0
+    
+    words = hypothesis.split()
+    if len(words) < n:
+        return len(words)  # Just return word count for very short texts
+    
+    # Count n-grams
+    ngrams = []
+    for i in range(len(words) - n + 1):
+        ngram = ' '.join(words[i:i+n])
+        ngrams.append(ngram)
+    
+    # More unique n-grams might indicate better fluency
+    unique_ngrams = len(set(ngrams))
+    total_ngrams = len(ngrams)
+    
+    # Score is a combination of length and n-gram variety
+    score = len(words) + unique_ngrams/max(1, total_ngrams) * 5
+    return score
+
+# N-best LM ranking approach
+def get_best_hypothesis_lm(hypotheses):
+    """Choose the best hypothesis using a simple language model approach"""
+    if not hypotheses:
+        return ""
+    
+    # Convert to list if it's not already
+    if isinstance(hypotheses, str):
+        return hypotheses
+    
+    # Ensure we have a list of strings
+    hypothesis_list = []
+    for h in hypotheses:
+        if isinstance(h, str):
+            hypothesis_list.append(preprocess_text(h))
+    
+    if not hypothesis_list:
+        return ""
+    
+    # Score each hypothesis and choose the best one
+    scores = [(score_hypothesis(h), h) for h in hypothesis_list]
+    best_hypothesis = max(scores, key=lambda x: x[0])[1]
+    return best_hypothesis
+
+# N-best correction approach
+def correct_hypotheses(hypotheses):
+    """Simple n-best correction by voting on words"""
+    if not hypotheses:
+        return ""
+    
+    # Convert to list if it's not already
+    if isinstance(hypotheses, str):
+        return hypotheses
+    
+    # Ensure we have a list of strings
+    hypothesis_list = []
+    for h in hypotheses:
+        if isinstance(h, str):
+            hypothesis_list.append(preprocess_text(h))
+    
+    if not hypothesis_list:
+        return ""
+    
+    # Split hypotheses into words
+    word_lists = [h.split() for h in hypothesis_list]
+    
+    # Find the most common length
+    lengths = [len(words) for words in word_lists]
+    if not lengths:
+        return ""
+    
+    most_common_length = Counter(lengths).most_common(1)[0][0]
+    
+    # Only consider hypotheses with the most common length
+    filtered_word_lists = [words for words in word_lists if len(words) == most_common_length]
+    
+    if not filtered_word_lists:
+        # Fall back to the longest hypothesis if filtering removed everything
+        return max(hypothesis_list, key=len)
+    
+    # Vote on each word position
+    corrected_words = []
+    for i in range(most_common_length):
+        position_words = [words[i] for words in filtered_word_lists]
+        most_common_word = Counter(position_words).most_common(1)[0][0]
+        corrected_words.append(most_common_word)
+    
+    # Join the corrected words
+    return ' '.join(corrected_words)
+
 # Fix the Levenshtein distance calculation to avoid dependence on jiwer internals
 def calculate_simple_wer(reference, hypothesis):
     """Calculate WER using a simple word-based approach"""
@@ -67,10 +163,10 @@ def calculate_simple_wer(reference, hypothesis):
         return 1.0
     return float(distance) / float(len(ref_words))
 
-# Calculate WER for a group of examples
-def calculate_wer(examples):
+# Calculate WER for a group of examples with multiple methods
+def calculate_wer_methods(examples):
     if not examples:
-        return 0.0
+        return 0.0, 0.0, 0.0
     
     try:
         # Check if examples is a Dataset or a list
@@ -83,7 +179,7 @@ def calculate_wer(examples):
             example = examples[0]
         else:
             print("No examples found")
-            return np.nan
+            return np.nan, np.nan, np.nan
             
         print("\n===== EXAMPLE DATA INSPECTION =====")
         print(f"Keys in example: {example.keys()}")
@@ -101,7 +197,10 @@ def calculate_wer(examples):
                 print(f"Hypothesis field '{field}' found with value: {str(example[field])[:100]}...")
         
         # Process each example in the dataset
-        wer_values = []
+        wer_values_no_lm = []
+        wer_values_lm_ranking = []
+        wer_values_n_best_correction = []
+        
         valid_count = 0
         skipped_count = 0
         
@@ -115,10 +214,19 @@ def calculate_wer(examples):
         
         for i, ex in enumerate(items_to_process):
             try:
-                # Try to get transcription and input1
+                # Get reference transcription
                 transcription = ex.get("transcription")
+                if not transcription or not isinstance(transcription, str):
+                    skipped_count += 1
+                    continue
+                
+                # Process the reference
+                reference = preprocess_text(transcription)
+                if not reference:
+                    skipped_count += 1
+                    continue
                 
-                # First try input1, then use first element from hypothesis if available
+                # Get 1-best hypothesis for baseline
                 input1 = ex.get("input1")
                 if input1 is None and "hypothesis" in ex and ex["hypothesis"]:
                     if isinstance(ex["hypothesis"], list) and len(ex["hypothesis"]) > 0:
@@ -126,58 +234,89 @@ def calculate_wer(examples):
                     elif isinstance(ex["hypothesis"], str):
                         input1 = ex["hypothesis"]
                 
-                # Print debug info for a few examples
-                if i < 3:
-                    print(f"\nExample {i} inspection:")
-                    print(f"  transcription: {transcription}")
-                    print(f"  input1: {input1}")
-                    print(f"  type checks: transcription={type(transcription)}, input1={type(input1)}")
+                # Get n-best hypotheses for other methods
+                n_best_hypotheses = ex.get("hypothesis", [])
                 
-                # Skip if either field is missing
-                if transcription is None or input1 is None:
-                    skipped_count += 1
-                    if i < 3:
-                        print(f"  SKIPPED: Missing field (transcription={transcription is None}, input1={input1 is None})")
-                    continue
+                # Process and evaluate all methods
                 
-                # Skip if either field is empty after preprocessing
-                reference = preprocess_text(transcription)
-                hypothesis = preprocess_text(input1)
+                # Method 1: No LM (1-best ASR output)
+                if input1 and isinstance(input1, str):
+                    no_lm_hyp = preprocess_text(input1)
+                    if no_lm_hyp:
+                        wer_no_lm = calculate_simple_wer(reference, no_lm_hyp)
+                        wer_values_no_lm.append(wer_no_lm)
                 
-                if not reference or not hypothesis:
-                    skipped_count += 1
-                    if i < 3:
-                        print(f"  SKIPPED: Empty after preprocessing (reference='{reference}', hypothesis='{hypothesis}')")
-                    continue
+                # Method 2: LM ranking (best of n-best)
+                if n_best_hypotheses:
+                    lm_best_hyp = get_best_hypothesis_lm(n_best_hypotheses)
+                    if lm_best_hyp:
+                        wer_lm = calculate_simple_wer(reference, lm_best_hyp)
+                        wer_values_lm_ranking.append(wer_lm)
+                
+                # Method 3: N-best correction (voting among n-best)
+                if n_best_hypotheses:
+                    corrected_hyp = correct_hypotheses(n_best_hypotheses)
+                    if corrected_hyp:
+                        wer_corrected = calculate_simple_wer(reference, corrected_hyp)
+                        wer_values_n_best_correction.append(wer_corrected)
                 
-                # Calculate WER for this pair
-                pair_wer = calculate_simple_wer(reference, hypothesis)
-                wer_values.append(pair_wer)
-                valid_count += 1
+                # Count as valid if at least one method worked
+                if (wer_values_no_lm and i == len(wer_values_no_lm) - 1) or \
+                   (wer_values_lm_ranking and i == len(wer_values_lm_ranking) - 1) or \
+                   (wer_values_n_best_correction and i == len(wer_values_n_best_correction) - 1):
+                    valid_count += 1
+                else:
+                    skipped_count += 1
                 
-                if i < 3:
-                    print(f"  VALID PAIR: reference='{reference}', hypothesis='{hypothesis}', WER={pair_wer:.4f}")
+                # Print debug info for a few examples
+                if i < 2:
+                    print(f"\nExample {i} inspection:")
+                    print(f"  Reference: '{reference}'")
+                    
+                    if input1 and isinstance(input1, str):
+                        no_lm_hyp = preprocess_text(input1)
+                        print(f"  No LM (1-best): '{no_lm_hyp}'")
+                        if no_lm_hyp:
+                            wer = calculate_simple_wer(reference, no_lm_hyp)
+                            print(f"  No LM WER: {wer:.4f}")
+                    
+                    if n_best_hypotheses:
+                        print(f"  N-best count: {len(n_best_hypotheses) if isinstance(n_best_hypotheses, list) else 'not a list'}")
+                        lm_best_hyp = get_best_hypothesis_lm(n_best_hypotheses)
+                        print(f"  LM ranking best: '{lm_best_hyp}'")
+                        if lm_best_hyp:
+                            wer = calculate_simple_wer(reference, lm_best_hyp)
+                            print(f"  LM ranking WER: {wer:.4f}")
+                        
+                        corrected_hyp = correct_hypotheses(n_best_hypotheses)
+                        print(f"  N-best correction: '{corrected_hyp}'")
+                        if corrected_hyp:
+                            wer = calculate_simple_wer(reference, corrected_hyp)
+                            print(f"  N-best correction WER: {wer:.4f}")
                 
             except Exception as ex_error:
                 print(f"Error processing example {i}: {str(ex_error)}")
                 skipped_count += 1
                 continue
         
-        # Calculate average WER
+        # Calculate average WER for each method
         print(f"\nProcessing summary: Valid pairs: {valid_count}, Skipped: {skipped_count}")
         
-        if not wer_values:
-            print("No valid pairs found for WER calculation")
-            return np.nan
+        no_lm_wer = np.mean(wer_values_no_lm) if wer_values_no_lm else np.nan
+        lm_ranking_wer = np.mean(wer_values_lm_ranking) if wer_values_lm_ranking else np.nan
+        n_best_correction_wer = np.mean(wer_values_n_best_correction) if wer_values_n_best_correction else np.nan
+        
+        print(f"Calculated WERs:")
+        print(f"  No LM: {len(wer_values_no_lm)} pairs, avg WER: {no_lm_wer:.4f}")
+        print(f"  LM Ranking: {len(wer_values_lm_ranking)} pairs, avg WER: {lm_ranking_wer:.4f}")
+        print(f"  N-best Correction: {len(wer_values_n_best_correction)} pairs, avg WER: {n_best_correction_wer:.4f}")
         
-        avg_wer = np.mean(wer_values)
-        print(f"Calculated {len(wer_values)} pairs with average WER: {avg_wer:.4f}")
-        return avg_wer
+        return no_lm_wer, lm_ranking_wer, n_best_correction_wer
     
     except Exception as e:
         print(f"Error in calculate_wer: {str(e)}")
         print(traceback.format_exc())
-        return np.nan
+        return np.nan, np.nan, np.nan
 
 # Get WER metrics by source 
 def get_wer_metrics(dataset):
@@ -218,19 +357,23 @@ def get_wer_metrics(dataset):
                 
                 if count > 0:
                     print(f"\nCalculating WER for source {source} with {count} examples")
-                    wer = calculate_wer(examples)  # Now handles both lists and datasets
+                    no_lm_wer, lm_ranking_wer, n_best_wer = calculate_wer_methods(examples)
                 else:
-                    wer = np.nan
+                    no_lm_wer, lm_ranking_wer, n_best_wer = np.nan, np.nan, np.nan
                 
                 source_results[source] = {
                     "Count": count,
-                    "No LM Baseline": wer
+                    "No LM Baseline": no_lm_wer,
+                    "N-best LM Ranking": lm_ranking_wer,
+                    "N-best Correction": n_best_wer
                 }
             except Exception as e:
                 print(f"Error processing source {source}: {str(e)}")
                 source_results[source] = {
                     "Count": 0,
-                    "No LM Baseline": np.nan
+                    "No LM Baseline": np.nan,
+                    "N-best LM Ranking": np.nan,
+                    "N-best Correction": np.nan
                 }
         
         # Calculate overall metrics with a sample but excluding all_et05_real
@@ -243,26 +386,35 @@ def get_wer_metrics(dataset):
             # Sample for calculation
             sample_size = min(500, total_count)
             sample_dataset = filtered_dataset[:sample_size]
-            overall_wer = calculate_wer(sample_dataset)
+            no_lm_wer, lm_ranking_wer, n_best_wer = calculate_wer_methods(sample_dataset)
             
             source_results["OVERALL"] = {
                 "Count": total_count,
-                "No LM Baseline": overall_wer
+                "No LM Baseline": no_lm_wer,
+                "N-best LM Ranking": lm_ranking_wer,
+                "N-best Correction": n_best_wer
             }
         except Exception as e:
             print(f"Error calculating overall metrics: {str(e)}")
             print(traceback.format_exc())
             source_results["OVERALL"] = {
                 "Count": len(filtered_dataset),
-                "No LM Baseline": np.nan
+                "No LM Baseline": np.nan,
+                "N-best LM Ranking": np.nan,
+                "N-best Correction": np.nan
             }
         
         # Create a transposed DataFrame with metrics as rows and sources as columns
-        metrics = ["Count", "No LM Baseline"]
+        metrics = ["Count", "No LM Baseline", "N-best LM Ranking", "N-best Correction"]
         result_df = pd.DataFrame(index=metrics, columns=["Metric"] + all_sources + ["OVERALL"])
         
         # Add descriptive column
-        result_df["Metric"] = ["Number of Examples", "Word Error Rate (WER)"]
+        result_df["Metric"] = [
+            "Number of Examples", 
+            "Word Error Rate (No LM)",
+            "Word Error Rate (N-best LM Ranking)",
+            "Word Error Rate (N-best Correction)"
+        ]
         
         for source in all_sources + ["OVERALL"]:
             for metric in metrics:
@@ -284,14 +436,13 @@ def format_dataframe(df):
         # Use vectorized operations instead of apply
         df = df.copy()
         
-        # Find the row containing WER values (now with new index name)
-        wer_row_index = None
+        # Find the rows containing WER values
+        wer_row_indices = []
         for idx in df.index:
             if "WER" in idx or "Error Rate" in idx:
-                wer_row_index = idx
-                break
+                wer_row_indices.append(idx)
         
-        if wer_row_index:
+        for wer_row_index in wer_row_indices:
             # Convert to object type first to avoid warnings
             df.loc[wer_row_index] = df.loc[wer_row_index].astype(object)
             
@@ -323,7 +474,7 @@ def create_leaderboard():
 # Create the Gradio interface
 with gr.Blocks(title="ASR Text Correction Test Leaderboard") as demo:
     gr.Markdown("# ASR Text Correction Baseline WER Leaderboard (Test Data)")
-    gr.Markdown("Word Error Rate (WER) metrics for different speech sources with No Language Model baseline")
+    gr.Markdown("Word Error Rate (WER) metrics for different speech sources with multiple correction approaches")
     
     with gr.Row():
         refresh_btn = gr.Button("Refresh Leaderboard")