Spaces:
Running
Running
| import csv | |
| from collections import defaultdict | |
| import gradio as gr | |
| import pandas as pd | |
| from pathlib import Path | |
| def strip_colname(x): | |
| if x.startswith("score_"): | |
| return x[6:] | |
| return x | |
| INTRO = """The current leaderboard displays performance across all filtered directions based on the test subset of BOUQuET. | |
| The code for reproducing the evaluations or adding a new model is available at https://github.com/facebookresearch/bouquet! | |
| """ | |
| LANGS_COMMENT = """## Languages | |
| For the meaning of the language codes, please refer to the *Languages* tab. | |
| """ | |
| METRICS_EXPLANATION = """## Metrics | |
| 1. `metricx`: [google/metricx-24-hybrid-xl-v2p6](https://huggingface.co/google/metricx-24-hybrid-xl-v2p6) score based on both source and reference. **Attention: lower is better!** | |
| 2. `chrfpp`: ChrF++ score ([sacrebleu](https://github.com/mjpost/sacrebleu) implementation) based on reference. We report the mean of sentence-level scores instead of a system-level score. **Attention: values not directly comparable across different target languages!** | |
| 3. `glotlid`: probability, as predicted by the [GlotLID model](https://huggingface.co/cis-lmu/glotlid), that translation is in the target language. Note that about 20\% of BOUQuET languages are not supported by GlotLID. | |
| 4. `glotlid_x_metricx`: a composite score computed as `glotlid * sqrt(1 - metricx / 25)` to penalize MetricX for off-target translations; higher is better. We report the mean of the sentence-level scores. | |
| """ | |
| SYSTEMS_EXPLANATION = """## Systems | |
| Descriptions of the implementation of the systems will come out later. | |
| """ | |
| REVERSE_METRICS = {"metricx"} | |
| DEFAULT_METRIC = "glotlid_x_metricx" | |
| def leaderboard_tab(): | |
| # stats = pd.read_csv("data/benchmark_stats.tsv", sep="\t", quoting=csv.QUOTE_NONE) | |
| # stats.columns = [strip_colname(c) for c in stats.columns] | |
| stats_parts = [] | |
| for model_dir in Path("data/mt_benchmark").iterdir(): | |
| filename = model_dir / "scores.csv" | |
| if not filename.exists(): | |
| print(f"No {fn.name} in {model_dir.name} found; skipping") | |
| continue | |
| part = pd.read_csv(filename) | |
| part["system"] = model_dir.name | |
| part["level"] = "sentence_level" | |
| stats_parts.append(part) | |
| stats = pd.concat(stats_parts, ignore_index=True) | |
| metrics = ["chrfpp", "metricx", "glotlid_score", "glotlid_x_metricx"] | |
| systems = sorted(set(stats["system"])) | |
| levels = ["sentence_level"] | |
| # TODO: reintroduce "paragraph_level" | |
| ALL = "ALL" | |
| MEAN = "Average" | |
| BEST = "Best" | |
| XX2EN = "Everything-into-English" | |
| EN2XX = "English-into-Everything" | |
| lang_src2tgt = defaultdict(set) | |
| lang_tgt2src = defaultdict(set) | |
| langs_src = set() | |
| langs_tgt = set() | |
| for src_lang, tgt_lang in stats[["src_lang", "tgt_lang"]].drop_duplicates().values: | |
| lang_src2tgt[src_lang].add(tgt_lang) | |
| lang_tgt2src[tgt_lang].add(src_lang) | |
| langs_src.add(src_lang) | |
| langs_tgt.add(tgt_lang) | |
| langs_df = pd.read_csv("data/language_metadata.tsv", sep="\t") | |
| lang2name = {} | |
| for i, row in langs_df.iterrows(): | |
| code = row["Code"] | |
| lang2name[code] = row["Language"] | |
| if isinstance(row["Secondary ISO 639-3"], str) and len( | |
| row["Secondary ISO 639-3"] | |
| ): | |
| code = row["Secondary ISO 639-3"] + code[3:] | |
| lang2name[code] = row["Language"] | |
| for lang in langs_src.union(langs_tgt): | |
| if lang not in lang2name: | |
| print(f"Name not found for {lang}") | |
| def named_langs(langs_list): | |
| return [ | |
| (f"{lang} — {lang2name[lang]}", lang) if lang in lang2name else lang | |
| for lang in langs_list | |
| ] | |
| with gr.Tab("MT leaderboard"): | |
| gr.Markdown("# BOUQuET machine translation leaderboard") | |
| gr.Markdown(INTRO) | |
| gr.Markdown("## Systems ranking") | |
| # Inputs | |
| gr_level = gr.Dropdown(levels, value="sentence_level", label="Level") | |
| gr_src_lang = gr.Dropdown( | |
| [ALL] + named_langs(sorted(langs_src)), value=ALL, label="Source lang" | |
| ) | |
| gr_tgt_lang = gr.Dropdown( | |
| [ALL] + named_langs(sorted(langs_tgt)), value=ALL, label="Target lang" | |
| ) | |
| # Interactivity | |
| inputs = [gr_level, gr_src_lang, gr_tgt_lang] | |
| def get_lb(level, src_lang, tgt_lang): | |
| filtered = stats[stats["level"].eq(level)] | |
| if src_lang != ALL: | |
| filtered = filtered[filtered["src_lang"].eq(src_lang)] | |
| if tgt_lang != ALL: | |
| filtered = filtered[filtered["tgt_lang"].eq(tgt_lang)] | |
| means = ( | |
| filtered.groupby(["system"])[metrics] | |
| .mean() | |
| .reset_index() | |
| .sort_values(DEFAULT_METRIC, ascending=DEFAULT_METRIC in REVERSE_METRICS) | |
| ) | |
| means.columns = [strip_colname(c) for c in means.columns] | |
| styler = means.style.background_gradient().format(precision=4) | |
| return styler | |
| df_all = get_lb(*[inp.value for inp in inputs]) | |
| gr_df = gr.Dataframe(df_all) | |
| for inp in inputs: | |
| inp.change(fn=get_lb, inputs=inputs, outputs=gr_df) | |
| # Interdependecy of the controls | |
| def src2tgt(src_lang, tgt_lang): | |
| if src_lang == ALL: | |
| choices = [ALL] + named_langs(sorted(langs_tgt)) | |
| else: | |
| choices = [ALL] + named_langs(sorted(lang_src2tgt[src_lang])) | |
| return gr.update(choices=choices, value=tgt_lang) | |
| def tgt2src(src_lang, tgt_lang): | |
| if tgt_lang == ALL: | |
| choices = [ALL] + named_langs(sorted(langs_src)) | |
| else: | |
| choices = [ALL] + named_langs(sorted(lang_tgt2src[tgt_lang])) | |
| return gr.update(choices=choices, value=src_lang) | |
| gr_src_lang.input( | |
| fn=src2tgt, inputs=[gr_src_lang, gr_tgt_lang], outputs=gr_tgt_lang | |
| ) | |
| gr_tgt_lang.input( | |
| fn=tgt2src, inputs=[gr_src_lang, gr_tgt_lang], outputs=gr_src_lang | |
| ) | |
| gr.Markdown("## Languages difficulty") | |
| gr_system = gr.Dropdown( | |
| [MEAN, BEST] + systems, value=MEAN, label="Translation system" | |
| ) | |
| gr_direction = gr.Dropdown( | |
| [XX2EN, EN2XX], value=XX2EN, label="Translation direction" | |
| ) | |
| gr_metric = gr.Dropdown(metrics, label="Quality metric", value=DEFAULT_METRIC) | |
| gr_level2 = gr.Dropdown(levels, value="sentence_level", label="Level") | |
| bar_controls = [gr_system, gr_direction, gr_metric, gr_level2] | |
| def get_hist(system, direction, metric, level): | |
| # decide on the data to process | |
| if direction == EN2XX: | |
| direction_filter = stats["src_lang"].eq("eng_Latn") | |
| lang_col = "tgt_lang" | |
| else: | |
| direction_filter = stats["tgt_lang"].eq("eng_Latn") | |
| lang_col = "src_lang" | |
| if system in (MEAN, BEST): | |
| system_filter = stats["system"].astype(bool) | |
| else: | |
| system_filter = stats["system"].eq(system) | |
| subset = stats[system_filter & direction_filter & stats["level"].eq(level)] | |
| # Compute the means and update the plot | |
| grouped = subset.groupby(lang_col)[metric] | |
| if metric in REVERSE_METRICS: | |
| bests = grouped.min() | |
| best_sys = grouped.idxmin() | |
| else: | |
| bests = grouped.max() | |
| best_sys = grouped.idxmax() | |
| if system == BEST: | |
| means = bests | |
| else: | |
| means = grouped.mean() | |
| report = ( | |
| pd.DataFrame( | |
| { | |
| metric: means, | |
| "best_system": subset.loc[best_sys]["system"].values, | |
| } | |
| ) | |
| .sort_values(metric, ascending=(metric in REVERSE_METRICS)) | |
| .reset_index() | |
| ) | |
| report["lang_name"] = [lang2name.get(lang, "") for lang in report[lang_col]] | |
| tooltip_columns = ["lang_name", "best_system"] | |
| # do not return gr.BarPlot, because it gets wrongly cached. | |
| return dict( | |
| value=report, | |
| x=lang_col, | |
| y=metric, | |
| x_title=lang_col, | |
| x_label_angle=-90, | |
| height=500, | |
| sort="y" if metric in REVERSE_METRICS else "-y", | |
| tooltip=tooltip_columns, | |
| ) | |
| # Use gr.render to fully re-mount the BarPlot on every input change. | |
| # This avoids Gradio's BarPlot diff-update bug where changing x/y | |
| # leaves the frontend with a stale cached dataset | |
| def render_bar(system, direction, metric, level): | |
| if not metric: | |
| return | |
| gr.BarPlot(**get_hist(system, direction, metric, level)) | |
| gr.Markdown(METRICS_EXPLANATION) | |
| gr.Markdown(SYSTEMS_EXPLANATION) | |
| gr.Markdown(LANGS_COMMENT) | |