Init

.gitignore

@@ -1,175 +0,0 @@
-checkpoints/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-src/
-# C extensions
-*.so
-temp
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-/tmp
-/logs
-/*.json
-/*.jpg
-/*.ipynb
-/GenAI-Arena-hf-logs
-/3DGen-Arena-logs*
-/tmp*
-/arena_elo/results/**/*.jpg
-/arena_elo/results/**/*.png
-/arena_elo/6_04_log_results
-/arena_elo/update_elo_rating_6_04.sh

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "GenAI-Arena-hf-logs"]
+	path = GenAI-Arena-hf-logs
+	url = https://github.com/jdf-prog/GenAI-Arena-hf-logs.git

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/GenAI-Arena.iml

@@ -1,15 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="PyDocumentationSettings">
-    <option name="format" value="GOOGLE" />
-    <option name="myDocStringFormat" value="Google" />
-  </component>
-  <component name="TemplatesService">
-    <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
-  </component>
-</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/GenAI-Arena.iml" filepath="$PROJECT_DIR$/.idea/GenAI-Arena.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

README.md

@@ -4,44 +4,10 @@ emoji: 📈
 colorFrom: purple
 colorTo: pink
 sdk: gradio
-sdk_version: 4.41.0
-python_version: 3.12
+sdk_version: 4.21.0
 app_file: app.py
-pinned: true
+pinned: false
 license: mit
-tags:
-- arena
-- leaderboard
-short_description: Realtime Image/Video Gen AI Arena
 ---
 
-## Installation
-
-- for cuda 11.8
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
-pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu118
-pip install -r requirements.txt
-```
-- for cuda 12.1
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
-pip install -r requirements.txt
-```
-
-## Start Hugging Face UI
-```bash
-python app.py
-```
-
-## Start Log server
-```bash
-uvicorn serve.log_server:app --reload --port 22005 --host 0.0.0.0
-```
-
-## Update leaderboard
-```bash
-cd arena_elo && bash update_leaderboard.sh
-```
-
-Paper: arxiv.org/abs/2406.04485
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -2,7 +2,6 @@ import gradio as gr
 import os
 from serve.gradio_web import *
 from serve.gradio_web_image_editing import *
-from serve.gradio_web_video_generation import *
 from serve.leaderboard import build_leaderboard_tab
 from model.model_manager import ModelManager
 from pathlib import Path
@@ -24,11 +23,13 @@ def build_combine_demo(models, elo_results_file, leaderboard_table_file):
                     with gr.Tab("Generation Arena (side-by-side)", id=1):
                         build_side_by_side_ui_named(models)
 
-                    with gr.Tab("Generation Playground", id=2): #Direct Chat
+                    with gr.Tab("Generation Direct Chat", id=2):
                         build_single_model_ui(models, add_promotion_links=True)
                     if elo_results_file:
                         with gr.Tab("Generation Leaderboard", id=3):
                             build_leaderboard_tab(elo_results_file['t2i_generation'], leaderboard_table_file['t2i_generation'])
+                    with gr.Tab("About Us", id=4):
+                        build_about()
             
             with gr.Tab("Image Edition", id=5):
                 with gr.Tabs() as tabs_ie:
@@ -38,27 +39,13 @@ def build_combine_demo(models, elo_results_file, leaderboard_table_file):
                     with gr.Tab("Edition Arena (side-by-side)", id=6):
                         build_side_by_side_ui_named_ie(models)
 
-                    with gr.Tab("Edition Playground", id=7): #Direct Chat
+                    with gr.Tab("Edition Direct Chat", id=7):
                         build_single_model_ui_ie(models, add_promotion_links=True)
                     if elo_results_file:
                         with gr.Tab("Edition Leaderboard", id=8):
                             build_leaderboard_tab(elo_results_file['image_editing'], leaderboard_table_file['image_editing'])
-
-            with gr.Tab("Video Generation", id=10):
-                with gr.Tabs() as tabs_vg:
-                    with gr.Tab("Video Generation Arena (battle)", id=10):
-                        build_side_by_side_ui_anony_vg(models)
-
-                    with gr.Tab("Video Generation Arena (side-by-side)", id=11):
-                        build_side_by_side_ui_named_vg(models)
-
-                    with gr.Tab("Video Generation Playground", id=12): #Direct Chat
-                        build_single_model_ui_vg(models, add_promotion_links=True)
-                    if elo_results_file and 'video_generation' in elo_results_file:
-                        with gr.Tab("Video Generation Leaderboard", id=13):
-                            build_leaderboard_tab(elo_results_file['video_generation'], leaderboard_table_file['video_generation'])
-            with gr.Tab("About Us", id=4): 
-                build_about()
+                    with gr.Tab("About Us", id=9):
+                        build_about()
 
     return demo
 
@@ -76,8 +63,6 @@ def load_elo_results(elo_results_dir):
                 elo_results_file['t2i_generation'] = file
             elif 'image_editing' in file.name:
                 elo_results_file['image_editing'] = file
-            elif 'video_generation' in file.name:
-                elo_results_file['video_generation'] = file
             else:
                 raise ValueError(f"Unknown file name: {file.name}")
         for file in elo_results_dir.glob('*_leaderboard.csv'):
@@ -85,20 +70,17 @@ def load_elo_results(elo_results_dir):
                 leaderboard_table_file['t2i_generation'] = file
             elif 'image_editing' in file.name:
                 leaderboard_table_file['image_editing'] = file
-            elif 'video_generation' in file.name:
-                leaderboard_table_file['video_generation'] = file
             else:
                 raise ValueError(f"Unknown file name: {file.name}")
             
     return elo_results_file, leaderboard_table_file
     
 if __name__ == "__main__":
-    server_port = int(SERVER_PORT)
+    server_port = SERVER_PORT
     root_path = ROOT_PATH
     elo_results_dir = ELO_RESULTS_DIR
-    # models = ModelManager(enable_nsfw=False, do_pre_download=True, do_debug_packages=True)
-    models = ModelManager(enable_nsfw=False, do_pre_download=False, do_debug_packages=False)
+    models = ModelManager()
     
     elo_results_file, leaderboard_table_file = load_elo_results(elo_results_dir)
     demo = build_combine_demo(models, elo_results_file, leaderboard_table_file)
-    demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH)
+    demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH)

arena_elo/edition_model_info.json

@@ -0,0 +1,37 @@
+{
+    "CycleDiffusion": {
+        "Link": "https://github.com/ChenWu98/cycle-diffusion",
+        "License": "X11",
+        "Organization": "Carnegie Mellon University"
+    },
+    "PNP": {
+        "Link": "https://github.com/MichalGeyer/plug-and-play",
+        "License": "-",
+        "Organization": "Weizmann Institute of Science"
+    },
+    "InstructPix2Pix": {
+        "Link": "https://www.timothybrooks.com/instruct-pix2pix",
+        "License": "Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros",
+        "Organization": "University of California, Berkeley"
+    },
+    "Pix2PixZero": {
+        "Link": "https://pix2pixzero.github.io",
+        "License": "MIT License",
+        "Organization": "Carnegie Mellon University, Adobe Research"
+    },
+    "MagicBrush": {
+        "Link": "https://osu-nlp-group.github.io/MagicBrush",
+        "License": "CC-BY-4.0",
+        "Organization": "The Ohio State University, University of Waterloo"
+    },
+    "Prompt2prompt": {
+        "Link": "https://prompt-to-prompt.github.io",
+        "License": "Apache-2.0",
+        "Organization": "Google, Tel Aviv University"
+    },
+    "SDEdit": {
+        "Link": "https://sde-image-editing.github.io",
+        "License": "MIT License",
+        "Organization": "Stanford University"
+    }
+}

arena_elo/elo_rating/clean_battle_data.py

@@ -18,13 +18,46 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
 from tqdm import tqdm
 
 from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
-from .utils import detect_language, get_time_stamp_from_date, get_model_info
+from .utils import detect_language, get_time_stamp_from_date
 
 VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote"]
+IDENTITY_WORDS = [
+    "vicuna",
+    "lmsys",
+    "koala",
+    "uc berkeley",
+    "open assistant",
+    "laion",
+    "chatglm",
+    "chatgpt",
+    "gpt-4",
+    "openai",
+    "anthropic",
+    "claude",
+    "bard",
+    "palm",
+    "lamda",
+    "google",
+    "llama",
+    "qianwan",
+    "alibaba",
+    "mistral",
+    "zhipu",
+    "KEG lab",
+    "01.AI",
+    "AI2",
+    "Tülu",
+    "Tulu",
+    "NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.",
+    "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES.",
+    "API REQUEST ERROR. Please increase the number of max tokens.",
+    "**API REQUEST ERROR** Reason: The response was blocked.",
+    "**API REQUEST ERROR**",
+]
+
+for i in range(len(IDENTITY_WORDS)):
+    IDENTITY_WORDS[i] = IDENTITY_WORDS[i].lower()
 
-def parse_model_name(model_name):
-    return NotImplementedError()
-    return model_source, model_name, model_type
 
 def remove_html(raw):
     if raw.startswith("<h3>"):
@@ -44,19 +77,19 @@ def to_openai_format(messages):
 
 def replace_model_name(old_name, tstamp):
     replace_dict = {
-        "PlayGroundV2": "PlayGround V2",
-        "PlayGroundV2.5": "PlayGround V2.5",
-        "FluxTimestep": "FLUX1schnell",
-        "FluxGuidance": "FLUX1dev",
-        "CogVideoX": "CogVideoX-2B"
+        "bard": "palm-2",
+        "claude-v1": "claude-1",
+        "claude-instant-v1": "claude-instant-1",
+        "oasst-sft-1-pythia-12b": "oasst-pythia-12b",
+        "claude-2": "claude-2.0",
     }
+    if old_name in ["gpt-4", "gpt-3.5-turbo"]:
+        if tstamp > 1687849200:
+            return old_name + "-0613"
+        else:
+            return old_name + "-0314"
     if old_name in replace_dict:
-        old_name =  replace_dict[old_name]
-    if "Flux" in old_name:
-        print(f"Invalid model names: {old_name}")
-        exit(1)
-    model_info = get_model_info(old_name)
-    old_name = model_info.simple_name
+        return replace_dict[old_name]
     return old_name
 
 
@@ -72,27 +105,18 @@ def read_file(filename):
             break
         except FileNotFoundError:
             time.sleep(2)
-        except json.JSONDecodeError:
-            print(f"Error in reading {filename}")
-            print(row)
-            exit(0)
     return data
 
 
 def read_file_parallel(log_files, num_threads=16):
     data_all = []
-    if num_threads == 1:
-        for log_file in tqdm(log_files, desc="Reading"):
-            data_all.extend(read_file(log_file))
-        return data_all
-    else:
-        from multiprocessing import Pool
-
-        with Pool(num_threads) as p:
-            ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
-            for ret in ret_all:
-                data_all.extend(ret)
-        return data_all
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
 
 def load_image(image_path):
     try:
@@ -103,7 +127,7 @@ def load_image(image_path):
 def clean_battle_data(
     log_files, exclude_model_names, ban_ip_list=None, sanitize_ip=False, mode="simple", task_name="image_editing"
 ):
-    data = read_file_parallel(log_files, num_threads=1)
+    data = read_file_parallel(log_files, num_threads=16)
 
     convert_type = {
         "leftvote": "model_a",
@@ -121,7 +145,6 @@ def clean_battle_data(
     battles = []
     for row in tqdm(data, desc="Cleaning"):
         if row["models"][0] is None or row["models"][1] is None:
-            print(f"Invalid model names: {row['models']}")
             continue
 
         # Resolve model names
@@ -140,7 +163,6 @@ def clean_battle_data(
             models_public[1] == "" and models_public[0] != ""
         ):
             ct_invalid += 1
-            print(f"Invalid model names: {models_public}")
             continue
 
         if models_public[0] == "" or models_public[0] == "Model A":
@@ -151,82 +173,71 @@ def clean_battle_data(
             anony = False
             models = models_public
             if not models_public == models_hidden:
-                print(f"Model names mismatch: {models_public} vs {models_hidden}")
                 ct_invalid += 1
                 continue
-        
-        def preprocess_model_name(m):
-            if m == "Playground v2":
-                return 'playground_PlayGroundV2_generation'
-            if m == "Playground v2.5":
-                return 'playground_PlayGroundV2.5_generation'
-            return m
-        models = [preprocess_model_name(m) for m in models]
+
+        # # Detect langauge
+        # state = row["states"][0]
+        # if state["offset"] >= len(state["messages"]):
+        #     ct_invalid += 1
+        #     continue
+        # lang_code = detect_language(state["messages"][state["offset"]][1])
+
+        # # Drop conversations if the model names are leaked
+        # leaked_identity = False
+        # messages = ""
+        # for i in range(2):
+        #     state = row["states"][i]
+        #     for turn_idx, (role, msg) in enumerate(
+        #         state["messages"][state["offset"] :]
+        #     ):
+        #         if msg:
+        #             messages += msg.lower()
+        # for word in IDENTITY_WORDS:
+        #     if word in messages:
+        #         leaked_identity = True
+        #         break
+
+        # if leaked_identity:
+        #     ct_leaked_identity += 1
+        #     continue
 
         # Replace bard with palm
+        models = [replace_model_name(m, row["tstamp"]) for m in models]
         if task_name == "image_editing":
-            valid = True
-            for _model in models:
-                try:
-                    platform, model_name, task = _model.split("_")
-                except ValueError:
-                    valid = False
-                    break
-                if not (platform in ["playground", "imagenhub"] and task == "edition"):
-                    valid = False
-                    break
-            if not valid:
+            if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
+                # print(f"Invalid model names: {models}")
                 ct_invalid += 1
                 continue
-            for i, _model in enumerate(models):
-                platform, model_name, task = _model.split("_")
-                models[i] = model_name
-
+            models = [x[len("imagenhub_"):-len("_edition")] for x in models]
         elif task_name == "t2i_generation":
-            valid = True
-            for _model in models:
-                try:
-                    platform, model_name, task = _model.split("_")
-                except ValueError:
-                    valid = False
-                    break
-                if not (platform.lower() in ["playground", "imagenhub", 'fal'] and (task == "generation" or task == "text2image")):
-                    valid = False
-                    break
-            if not valid:
+            if not all("playground" in x.lower() or (x.startswith("imagenhub_") and x.endswith("_generation")) for x in models):
+                # print(f"Invalid model names: {models}")
                 ct_invalid += 1
                 continue
-            for i, _model in enumerate(models):
-                platform, model_name, task = _model.split("_")
-                models[i] = model_name
-            
-        elif task_name == "video_generation":
-            valid = True
-            for _model in models:
-                try:
-                    platform, model_name, task = _model.split("_")
-                except ValueError:
-                    valid = False
-                    break
-                if not (platform in ["videogenhub", "fal"] and task == "generation" or task == "text2video"):
-                    valid = False
-                    break
-            if not valid:
-                ct_invalid += 1
-                continue
-            for i, _model in enumerate(models):
-                platform, model_name, task = _model.split("_")
-                models[i] = model_name
+            # models = [x[len("imagenhub_"):-len("_generation")] for x in models]
+            for i, model_name in enumerate(models):
+                if model_name.startswith("imagenhub_"):
+                    models[i] = model_name[len("imagenhub_"):-len("_generation")]
 
         else:
             raise ValueError(f"Invalid task_name: {task_name}")
         
-        models = [replace_model_name(m, row["tstamp"]) for m in models]
-        
         # Exclude certain models
         if exclude_model_names and any(x in exclude_model_names for x in models):
             ct_invalid += 1
             continue
+        
+        # if models[0] not in model_infos or models[1] not in model_infos:
+        #     continue
+
+        # # Exclude votes before the starting date
+        # if model_infos and (model_infos[models[0]]["starting_from"] > row["tstamp"] or model_infos[models[1]]["starting_from"] > row["tstamp"]):
+        #     print(f"Invalid vote before the valid starting date for {models[0]} and {models[1]}")
+        #     ct_invalid += 1
+        #     continue
+        
+        
 
         if mode == "conv_release":
             # assert the two images are the same
@@ -251,6 +262,14 @@ def clean_battle_data(
                 continue
         
         
+        question_id = row["states"][0]["conv_id"]
+        # conversation_a = to_openai_format(
+        #     row["states"][0]["messages"][row["states"][0]["offset"] :]
+        # )
+        # conversation_b = to_openai_format(
+        #     row["states"][1]["messages"][row["states"][1]["offset"] :]
+        # )
+
         ip = row["ip"]
         if ip not in all_ips:
             all_ips[ip] = {"ip": ip, "count": 0, "sanitized_id": len(all_ips)}
@@ -262,45 +281,21 @@ def clean_battle_data(
 
         if ban_ip_list is not None and ip in ban_ip_list:
             ct_banned += 1
-            print(f"User {user_id} is banned")
             continue
-        required_keys_each_task = {
-            "image_editing": ["source_prompt", "target_prompt", "instruct_prompt"],
-            "t2i_generation": ["prompt"],
-            "video_generation": ["prompt"]
-        }
-        
-        model_a_inputs = row["states"][0].copy()
-        # pop conv_id and model_name
-        model_a_inputs.pop("conv_id")
-        model_a_inputs.pop("model_name")
-        model_b_inputs = row["states"][1].copy()
-        model_b_inputs.pop("conv_id")
-        model_b_inputs.pop("model_name")
-        for key in model_a_inputs:
-            if not (key in model_b_inputs and model_a_inputs[key] == model_b_inputs[key]):
-                print(f"Inconsistent inputs: {model_a_inputs} vs {model_b_inputs}")
-                ct_invalid += 1
-                continue
-        if mode == "conv_release":
-            if any(key not in model_a_inputs for key in required_keys_each_task[task_name]):
-                print(f"Missing required keys: {model_a_inputs}, {required_keys_each_task[task_name]}")
-                ct_invalid += 1
-                continue
-            
-        inputs = model_a_inputs
+
         # Save the results
         battles.append(
             dict(
-                model_a_conv_id=row["states"][0]["conv_id"],
-                model_b_conv_id=row["states"][1]["conv_id"],
-                inputs=inputs,
+                question_id=question_id,
                 model_a=models[0],
                 model_b=models[1],
-                vote_type=row["type"],
                 winner=convert_type[row["type"]],
                 judge=f"arena_user_{user_id}",
+                # conversation_a=conversation_a,
+                # conversation_b=conversation_b,
+                # turn=len(conversation_a) // 2,
                 anony=anony,
+                # language=lang_code,
                 tstamp=row["tstamp"],
             )
         )
@@ -337,7 +332,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--mode", type=str, choices=["simple", "conv_release"], default="simple"
     )
-    parser.add_argument("--task_name", type=str, default="image_editing", choices=["image_editing", "t2i_generation", "video_generation"])
+    parser.add_argument("--task_name", type=str, default="image_editing", choices=["image_editing", "t2i_generation"])
     parser.add_argument("--exclude-model-names", type=str, nargs="+")
     parser.add_argument("--ban-ip-file", type=str)
     parser.add_argument("--sanitize-ip", action="store_true", default=False)
@@ -355,19 +350,27 @@ if __name__ == "__main__":
     ).strftime("%Y%m%d")
 
     if args.mode == "simple":
-        # for x in battles:
-        #     for key in [
-        #         "conversation_a",
-        #         "conversation_b",
-        #         "question_id",
-        #     ]:
-        #         if key in x:
-        #             del x[key]
+        for x in battles:
+            for key in [
+                "conversation_a",
+                "conversation_b",
+                "question_id",
+            ]:
+                if key in x:
+                    del x[key]
         print("Samples:")
         for i in range(min(4, len(battles))):
             print(battles[i])
         output = f"clean_battle_{args.task_name}_{cutoff_date}.json"
     elif args.mode == "conv_release":
+        # new_battles = []
+        # for x in battles:
+        #     if not x["anony"]:
+        #         continue
+        #     for key in []:
+        #         del x[key]
+        #     new_battles.append(x)
+        # battles = new_battles
         output = f"clean_battle_{args.task_name}_conv_{cutoff_date}.json"
 
     with open(output, "w") as fout:

arena_elo/elo_rating/elo_analysis.py

@@ -11,9 +11,9 @@ import pandas as pd
 import plotly.express as px
 from tqdm import tqdm
 
+from .model_registry import get_model_info
 from .basic_stats import get_log_files
 from .clean_battle_data import clean_battle_data
-from .utils import get_model_info
         
 pd.options.display.float_format = "{:.2f}".format
 
@@ -214,9 +214,8 @@ def visualize_average_win_rate(battles, limit_show_number):
         width=700,
     )
     fig.update_layout(
-        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False,
+        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False
     )
-    fig.update_traces(textfont_size=16)
     return fig
 
 
@@ -247,7 +246,6 @@ def visualize_bootstrap_elo_rating(df, df_final, limit_show_number):
         width=700,
     )
     fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
-    fig.update_traces(textfont_size=16)
     return fig
 
 
@@ -340,7 +338,6 @@ if __name__ == "__main__":
         "--rating-system", type=str, choices=["bt", "elo"], default="bt"
     )
     parser.add_argument("--exclude-tie", action="store_true", default=False)
-    parser.add_argument("--min_num_battles_per_model", type=int, default=25)
     args = parser.parse_args()
 
     np.random.seed(42)
@@ -352,23 +349,7 @@ if __name__ == "__main__":
         # Read data from all log files
         log_files = get_log_files(args.max_num_files)
         battles = clean_battle_data(log_files)
-        
-    if args.min_num_battles_per_model:
-        num_battles_per_model = defaultdict(int)
-        # use pd
-        for _, battle in battles.iterrows():
-            num_battles_per_model[battle["model_a"]] += 1
-            num_battles_per_model[battle["model_b"]] += 1
-        to_remove_models = [
-            model for model, num_battles in num_battles_per_model.items() if num_battles < args.min_num_battles_per_model
-        ]
-        battles_with_enough_battles = battles[
-            ~battles["model_a"].isin(to_remove_models) & ~battles["model_b"].isin(to_remove_models)
-        ]
-        print(f"Remove models with less than {args.min_num_battles_per_model} battles: {to_remove_models}")
-        print(f"Number of battles: {len(battles)} -> {len(battles_with_enough_battles)}")
-        battles = battles_with_enough_battles
-        
+
     anony_results = report_elo_analysis_results(
         battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=True
     )
@@ -381,22 +362,9 @@ if __name__ == "__main__":
     pretty_print_elo_rating(anony_results["elo_rating_online"])
     print("# Median")
     pretty_print_elo_rating(anony_results["elo_rating_final"])
-    print(f"Annoy last update : {anony_results['last_updated_datetime']}")
-    print(f"Full last update : {full_results['last_updated_datetime']}")
-    
-    
-    # # save heatmap results in the same directory of the cleaned battle file
-    win_fraction_heatmap_file = args.clean_battle_file.replace(".json", "_win_fraction_heatmap.jpg")
-    battle_count_heatmap_file = args.clean_battle_file.replace(".json", "_battle_count_heatmap.jpg")
-    average_win_rate_bar_file = args.clean_battle_file.replace(".json", "_average_win_rate_bar.jpg")
-    bootstrap_elo_rating_file = args.clean_battle_file.replace(".json", "_bootstrap_elo_rating.jpg")
-    anony_results["win_fraction_heatmap"].write_image(win_fraction_heatmap_file)
-    anony_results["battle_count_heatmap"].write_image(battle_count_heatmap_file)
-    anony_results["average_win_rate_bar"].write_image(average_win_rate_bar_file)
-    anony_results["bootstrap_elo_rating"].write_image(bootstrap_elo_rating_file)
-    
+    print(f"last update : {anony_results['last_updated_datetime']}")
 
-    last_updated_tstamp = full_results["last_updated_tstamp"]
+    last_updated_tstamp = anony_results["last_updated_tstamp"]
     cutoff_date = datetime.datetime.fromtimestamp(
         last_updated_tstamp, tz=timezone("US/Pacific")
     ).strftime("%Y%m%d")
@@ -408,6 +376,3 @@ if __name__ == "__main__":
     }
     with open(f"elo_results_{cutoff_date}.pkl", "wb") as fout:
         pickle.dump(results, fout)
-
-    with open("cut_off_date.txt", "w") as fout:
-        fout.write(cutoff_date)

arena_elo/elo_rating/generate_leaderboard.py

@@ -2,12 +2,15 @@ import fire
 import json
 import pandas as pd
 import pickle
-from .utils import get_model_info
+
 
 def main(
-    elo_rating_pkl: str,
-    output_csv: str
+        model_info_file: str,
+        elo_rating_pkl: str,
+        output_csv: str
 ):
+    model_info = json.load(open(model_info_file))
+
     with open(elo_rating_pkl, "rb") as fin:
         elo_rating_results = pickle.load(fin)
 
@@ -16,23 +19,19 @@ def main(
     anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
     full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
 
-    print(anony_leaderboard_data)
     # Model,MT-bench (score),Arena Elo rating,MMLU,License,Link
     fields = ["key", "Model", "Arena Elo rating (anony)", "Arena Elo rating (full)", "License", "Organization", "Link"]
     # set Organization and license to empty for now
     all_models = anony_leaderboard_data.index.tolist()
 
-    model_info = {}
     for model in all_models:
-    
-        registered_model_info = get_model_info(model)
-        model_info[model] = {
-            "key": model,
-            "Model": model,
-            "License": registered_model_info.license,
-            "Organization": registered_model_info.organization,
-            "Link": registered_model_info.link
-        }
+        if not model in model_info:
+            model_info[model] = {}
+            model_info[model]["License"] = "N/A"
+            model_info[model]["Organization"] = "N/A"
+            model_info[model]["Link"] = "N/A"
+        model_info[model]["Model"] = model
+        model_info[model]["key"] = model
 
         if model in anony_leaderboard_data.index:
             model_info[model]["Arena Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]
@@ -43,6 +42,10 @@ def main(
             model_info[model]["Arena Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
         else:
             model_info[model]["Arena Elo rating (full)"] = 0
+        # if model in anony_leaderboard_data.index:
+        #     model_info[model]["Arena Elo rating"] = anony_leaderboard_data.loc[model, "rating"]
+        # else:
+        #     model_info[model]["Arena Elo rating"] = 0
 
     final_model_info = {}
     for model in model_info:

arena_elo/elo_rating/model_registry.py

@@ -0,0 +1,578 @@
+"""Additional information of the models."""
+from collections import namedtuple, OrderedDict
+from typing import List
+
+
+ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description"])
+
+
+model_info = OrderedDict()
+
+
+def register_model_info(
+    full_names: List[str], simple_name: str, link: str, description: str
+):
+    info = ModelInfo(simple_name, link, description)
+
+    for full_name in full_names:
+        model_info[full_name] = info
+
+
+def get_model_info(name: str) -> ModelInfo:
+    if name in model_info:
+        return model_info[name]
+    else:
+        # To fix this, please use `register_model_info` to register your model
+        return ModelInfo(
+            name, "", "Register the description at arena.model/model_registry.py"
+        )
+
+
+register_model_info(
+    [
+        "IEITYuan/Yuan2-2B-Janus-hf",
+        "IEITYuan/Yuan2-2B-hf",
+        "IEITYuan/Yuan2-51B-hf",
+        "IEITYuan/Yuan2-102B-hf",
+    ],
+    "IEIT-Yuan2",
+    "https://github.com/IEIT-Yuan/Yuan-2.0",
+    "Yuan2.0 is a new generation Fundamental Large Language Model developed by IEIT System.",
+)
+
+register_model_info(
+    ["mixtral-8x7b-instruct-v0.1", "mistral-7b-instruct"],
+    "Mixtral of experts",
+    "https://mistral.ai/news/mixtral-of-experts/",
+    "A Mixture-of-Experts model by Mistral AI",
+)
+
+register_model_info(
+    ["gemini-pro"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["gemini-pro-vision"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["solar-10.7b-instruct-v1.0"],
+    "SOLAR-10.7B-Instruct",
+    "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0",
+    "A model trained using depth up-scaling by Upstage AI",
+)
+
+register_model_info(
+    ["gpt-4-turbo"],
+    "GPT-4-Turbo",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4-vision-preview"],
+    "gpt-4-vision-preview",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4(V) by OpenAI",
+)
+
+register_model_info(
+    ["gpt-3.5-turbo", "gpt-3.5-turbo-0314", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-1106"],
+    "GPT-3.5",
+    "https://platform.openai.com/docs/models/gpt-3-5",
+    "GPT-3.5-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4", "gpt-4-0314", "gpt-4-0613"],
+    "GPT-4",
+    "https://openai.com/research/gpt-4",
+    "GPT-4 by OpenAI",
+)
+
+register_model_info(
+    ["claude-2.1", "claude-2.0"],
+    "Claude",
+    "https://www.anthropic.com/index/claude-2",
+    "Claude 2 by Anthropic",
+)
+
+register_model_info(
+    ["claude-1"],
+    "Claude",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude 1 by Anthropic",
+)
+
+register_model_info(
+    ["claude-instant-1", "claude-instant-1.2"],
+    "Claude Instant",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude Instant by Anthropic",
+)
+
+register_model_info(
+    ["pplx-70b-online", "pplx-7b-online"],
+    "pplx-online-llms",
+    "https://blog.perplexity.ai/blog/introducing-pplx-online-llms",
+    "Online LLM API by Perplexity AI",
+)
+
+register_model_info(
+    ["openhermes-2.5-mistral-7b"],
+    "OpenHermes-2.5-Mistral-7B",
+    "https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B",
+    "a mistral-based model fine-tuned on 1M GPT-4 outputs",
+)
+
+register_model_info(
+    ["starling-lm-7b-alpha"],
+    "Starling-LM-7B-alpha",
+    "https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha",
+    "an open model trained using RLAIF by Berkeley",
+)
+
+register_model_info(
+    ["tulu-2-dpo-70b"],
+    "Tulu 2",
+    "https://huggingface.co/allenai/tulu-2-dpo-70b",
+    "an instruction and RLHF model by UW/AllenAI",
+)
+
+register_model_info(
+    ["yi-34b-chat", "yi-6b-chat"],
+    "Yi-Chat",
+    "https://huggingface.co/01-ai/Yi-34B-Chat",
+    "A large language model by 01 AI",
+)
+
+register_model_info(
+    ["llama-2-70b-chat", "llama-2-34b-chat", "llama-2-13b-chat", "llama-2-7b-chat"],
+    "Llama 2",
+    "https://ai.meta.com/llama/",
+    "open foundation and fine-tuned chat models by Meta",
+)
+
+register_model_info(
+    [
+        "vicuna-33b",
+        "vicuna-33b-v1.3",
+        "vicuna-13b",
+        "vicuna-13b-v1.3",
+        "vicuna-7b",
+        "vicuna-7b-v1.3",
+    ],
+    "Vicuna",
+    "https://lmsys.org/blog/2023-03-30-vicuna/",
+    "a chat assistant fine-tuned on user-shared conversations by LMSYS",
+)
+
+register_model_info(
+    ["chatglm3-6b", "chatglm2-6b", "chatglm-6b"],
+    "ChatGLM",
+    "https://chatglm.cn/blog",
+    "an open bilingual dialogue language model by Tsinghua University",
+)
+
+register_model_info(
+    ["openchat-3.5"],
+    "OpenChat 3.5",
+    "https://github.com/imoneoi/openchat",
+    "an open model fine-tuned on Mistral-7B using C-RLFT",
+)
+
+register_model_info(
+    ["tenyxchat-7b-v1"],
+    "TenyxChat-7B",
+    "https://huggingface.co/tenyx/TenyxChat-7B-v1",
+    "an open model DPO trained on top of OpenChat-3.5 using Tenyx fine-tuning",
+)
+
+register_model_info(
+    ["zephyr-7b-beta", "zephyr-7b-alpha"],
+    "Zephyr",
+    "https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha",
+    "a chatbot fine-tuned from Mistral by Hugging Face",
+)
+
+register_model_info(
+    ["notus-7b-v1"],
+    "Notus",
+    "https://huggingface.co/argilla/notus-7b-v1",
+    "a chatbot fine-tuned from Zephyr SFT by Argilla",
+)
+
+register_model_info(
+    ["catppt"],
+    "CatPPT",
+    "https://huggingface.co/rishiraj/CatPPT",
+    "a chatbot fine-tuned from a SLERP merged model by Rishiraj Acharya",
+)
+
+register_model_info(
+    ["TinyLlama"],
+    "TinyLlama",
+    "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "The TinyLlama project is an open endeavor to pretrain a 1.1B Llama model on 3 trillion tokens.",
+)
+
+register_model_info(
+    ["qwen-14b-chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-14B-Chat",
+    "a large language model by Alibaba Cloud",
+)
+
+register_model_info(
+    ["codellama-34b-instruct", "codellama-13b-instruct", "codellama-7b-instruct"],
+    "Code Llama",
+    "https://ai.meta.com/blog/code-llama-large-language-model-coding/",
+    "open foundation models for code by Meta",
+)
+
+register_model_info(
+    ["wizardlm-70b", "wizardlm-30b", "wizardlm-13b"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM",
+    "an instruction-following LLM using evol-instruct by Microsoft",
+)
+
+register_model_info(
+    ["wizardcoder-15b-v1.0"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder",
+    "Empowering Code Large Language Models with Evol-Instruct",
+)
+
+register_model_info(
+    ["mpt-7b-chat", "mpt-30b-chat"],
+    "MPT-Chat",
+    "https://www.mosaicml.com/blog/mpt-30b",
+    "a chatbot fine-tuned from MPT by MosaicML",
+)
+
+register_model_info(
+    ["guanaco-33b", "guanaco-65b"],
+    "Guanaco",
+    "https://github.com/artidoro/qlora",
+    "a model fine-tuned with QLoRA by UW",
+)
+
+register_model_info(
+    ["gpt4all-13b-snoozy"],
+    "GPT4All-Snoozy",
+    "https://github.com/nomic-ai/gpt4all",
+    "a finetuned LLaMA model on assistant style data by Nomic AI",
+)
+
+register_model_info(
+    ["koala-13b"],
+    "Koala",
+    "https://bair.berkeley.edu/blog/2023/04/03/koala",
+    "a dialogue model for academic research by BAIR",
+)
+
+register_model_info(
+    ["RWKV-4-Raven-14B"],
+    "RWKV-4-Raven",
+    "https://huggingface.co/BlinkDL/rwkv-4-raven",
+    "an RNN with transformer-level LLM performance",
+)
+
+register_model_info(
+    ["alpaca-13b"],
+    "Alpaca",
+    "https://crfm.stanford.edu/2023/03/13/alpaca.html",
+    "a model fine-tuned from LLaMA on instruction-following demonstrations by Stanford",
+)
+
+register_model_info(
+    ["oasst-pythia-12b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["oasst-sft-7-llama-30b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["palm-2"],
+    "PaLM 2 Chat",
+    "https://cloud.google.com/vertex-ai/docs/release-notes#May_10_2023",
+    "PaLM 2 for Chat (chat-bison@001) by Google",
+)
+
+register_model_info(
+    ["llama-7b", "llama-13b"],
+    "LLaMA",
+    "https://arxiv.org/abs/2302.13971",
+    "open and efficient foundation language models by Meta",
+)
+
+register_model_info(
+    ["open-llama-7b-v2-open-instruct", "open-llama-7b-open-instruct"],
+    "Open LLaMa (Open Instruct)",
+    "https://medium.com/vmware-data-ml-blog/starter-llm-for-the-enterprise-instruction-tuning-openllama-7b-d05fc3bbaccc",
+    "Open LLaMa fine-tuned on instruction-following data by VMware",
+)
+
+register_model_info(
+    ["dolly-v2-12b"],
+    "Dolly",
+    "https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm",
+    "an instruction-tuned open large language model by Databricks",
+)
+
+register_model_info(
+    ["stablelm-tuned-alpha-7b"],
+    "StableLM",
+    "https://github.com/stability-AI/stableLM",
+    "Stability AI language models",
+)
+
+register_model_info(
+    ["codet5p-6b"],
+    "CodeT5p-6b",
+    "https://huggingface.co/Salesforce/codet5p-6b",
+    "Code completion model released by Salesforce",
+)
+
+register_model_info(
+    ["fastchat-t5-3b", "fastchat-t5-3b-v1.0"],
+    "FastChat-T5",
+    "https://huggingface.co/lmsys/fastchat-t5-3b-v1.0",
+    "a chat assistant fine-tuned from FLAN-T5 by LMSYS",
+)
+
+register_model_info(
+    ["phoenix-inst-chat-7b"],
+    "Phoenix-7B",
+    "https://huggingface.co/FreedomIntelligence/phoenix-inst-chat-7b",
+    "a multilingual chat assistant fine-tuned from Bloomz to democratize ChatGPT across languages by CUHK(SZ)",
+)
+
+register_model_info(
+    ["realm-7b-v1"],
+    "ReaLM",
+    "https://github.com/FreedomIntelligence/ReaLM",
+    "A chatbot fine-tuned from LLaMA2 with data generated via iterative calls to UserGPT and ChatGPT by CUHK(SZ) and SRIBD.",
+)
+
+register_model_info(
+    ["billa-7b-sft"],
+    "BiLLa-7B-SFT",
+    "https://huggingface.co/Neutralzz/BiLLa-7B-SFT",
+    "an instruction-tuned bilingual LLaMA with enhanced reasoning ability by an independent researcher",
+)
+
+register_model_info(
+    ["h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2"],
+    "h2oGPT-GM-7b",
+    "https://huggingface.co/h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2",
+    "an instruction-tuned OpenLLaMA with enhanced conversational ability by H2O.ai",
+)
+
+register_model_info(
+    ["baize-v2-7b", "baize-v2-13b"],
+    "Baize v2",
+    "https://github.com/project-baize/baize-chatbot#v2",
+    "A chatbot fine-tuned from LLaMA with ChatGPT self-chat data and Self-Disillation with Feedback (SDF) by UCSD and SYSU.",
+)
+
+register_model_info(
+    [
+        "airoboros-l2-7b-2.1",
+        "airoboros-l2-13b-2.1",
+        "airoboros-c34b-2.1",
+        "airoboros-l2-70b-2.1",
+    ],
+    "airoboros",
+    "https://huggingface.co/jondurbin/airoboros-l2-70b-2.1",
+    "an instruction-tuned LlaMa model tuned with 100% synthetic instruction-response pairs from GPT4",
+)
+
+register_model_info(
+    [
+        "spicyboros-7b-2.2",
+        "spicyboros-13b-2.2",
+        "spicyboros-70b-2.2",
+    ],
+    "spicyboros",
+    "https://huggingface.co/jondurbin/spicyboros-70b-2.2",
+    "de-aligned versions of the airoboros models",
+)
+
+register_model_info(
+    ["Robin-7b-v2", "Robin-13b-v2", "Robin-33b-v2"],
+    "Robin-v2",
+    "https://huggingface.co/OptimalScale/robin-7b-v2-delta",
+    "A chatbot fine-tuned from LLaMA-7b, achieving competitive performance on chitchat, commonsense reasoning and instruction-following tasks, by OptimalScale, HKUST.",
+)
+
+register_model_info(
+    ["manticore-13b-chat"],
+    "Manticore 13B Chat",
+    "https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg",
+    "A chatbot fine-tuned from LlaMa across several CoT and chat datasets.",
+)
+
+register_model_info(
+    ["redpajama-incite-7b-chat"],
+    "RedPajama-INCITE-7B-Chat",
+    "https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat",
+    "A chatbot fine-tuned from RedPajama-INCITE-7B-Base by Together",
+)
+
+register_model_info(
+    [
+        "falcon-7b",
+        "falcon-7b-instruct",
+        "falcon-40b",
+        "falcon-40b-instruct",
+        "falcon-180b",
+        "falcon-180b-chat",
+    ],
+    "Falcon",
+    "https://huggingface.co/tiiuae/falcon-180B",
+    "TII's flagship series of large language models",
+)
+
+register_model_info(
+    ["tigerbot-7b-sft"],
+    "Tigerbot",
+    "https://huggingface.co/TigerResearch/tigerbot-7b-sft",
+    "TigerBot is a large-scale language model (LLM) with multiple languages and tasks.",
+)
+
+register_model_info(
+    ["internlm-chat-7b", "internlm-chat-7b-8k"],
+    "InternLM",
+    "https://huggingface.co/internlm/internlm-chat-7b",
+    "InternLM is a multi-language large-scale language model (LLM), developed by SHLAB.",
+)
+
+register_model_info(
+    ["Qwen-7B-Chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-7B-Chat",
+    "Qwen is a multi-language large-scale language model (LLM), developed by Damo Academy.",
+)
+
+register_model_info(
+    ["Llama2-Chinese-13b-Chat", "LLama2-Chinese-13B"],
+    "Llama2-Chinese",
+    "https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat",
+    "Llama2-Chinese is a multi-language large-scale language model (LLM), developed by FlagAlpha.",
+)
+
+register_model_info(
+    ["Chinese-Alpaca-2-7B", "Chinese-Alpaca-2-13B"],
+    "Chinese-Alpaca",
+    "https://huggingface.co/hfl/chinese-alpaca-2-13b",
+    "New extended Chinese vocabulary beyond Llama-2, open-sourcing the Chinese LLaMA-2 and Alpaca-2 LLMs.",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Instruct", "Vigogne-2-13B-Instruct"],
+    "Vigogne-Instruct",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-instruct",
+    "Vigogne-Instruct is a French large language model (LLM) optimized for instruction-following, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Chat", "Vigogne-2-13B-Chat"],
+    "Vigogne-Chat",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-chat",
+    "Vigogne-Chat is a French large language model (LLM) optimized for instruction-following and multi-turn dialogues, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["stable-vicuna-13B-HF"],
+    "stable-vicuna",
+    "https://huggingface.co/TheBloke/stable-vicuna-13B-HF",
+    "StableVicuna is a Vicuna model fine-tuned using RLHF via PPO on various conversational and instructional datasets.",
+)
+
+register_model_info(
+    ["deluxe-chat-v1", "deluxe-chat-v1.1", "deluxe-chat-v1.2"],
+    "DeluxeChat",
+    "",
+    "Deluxe Chat",
+)
+
+register_model_info(
+    [
+        "Xwin-LM-7B-V0.1",
+        "Xwin-LM-13B-V0.1",
+        "Xwin-LM-70B-V0.1",
+        "Xwin-LM-7B-V0.2",
+        "Xwin-LM-13B-V0.2",
+    ],
+    "Xwin-LM",
+    "https://github.com/Xwin-LM/Xwin-LM",
+    "Chat models developed by Xwin-LM team",
+)
+
+register_model_info(
+    ["lemur-70b-chat"],
+    "Lemur-Chat",
+    "https://huggingface.co/OpenLemur/lemur-70b-chat-v1",
+    "an openly accessible language model optimized for both natural language and coding capabilities ",
+)
+
+register_model_info(
+    ["Mistral-7B-OpenOrca"],
+    "Open-Orca",
+    "https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca",
+    "A fine-tune of [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) using [OpenOrca dataset](https://huggingface.co/datasets/Open-Orca/OpenOrca)",
+)
+
+register_model_info(
+    ["dolphin-2.2.1-mistral-7b"],
+    "dolphin-mistral",
+    "https://huggingface.co/ehartford/dolphin-2.2.1-mistral-7b",
+    "An uncensored fine-tuned Mistral 7B",
+)
+
+register_model_info(
+    [
+        "AquilaChat-7B",
+        "AquilaChat2-7B",
+        "AquilaChat2-34B",
+    ],
+    "Aquila-Chat",
+    "https://huggingface.co/BAAI/AquilaChat2-34B",
+    "Chat models developed by BAAI team",
+)
+
+register_model_info(
+    ["xDAN-L1-Chat-RL-v1"],
+    "xDAN-L1-Chat",
+    "https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1",
+    "A large language chat model created by xDAN-AI.",
+)
+
+register_model_info(
+    ["MetaMath-70B-V1.0", "MetaMath-7B-V1.0"],
+    "MetaMath",
+    "https://huggingface.co/meta-math",
+    "MetaMath is a finetune of Llama2 on [MetaMathQA](https://huggingface.co/datasets/meta-math/MetaMathQA) that specializes in mathematical reasoning.",
+)
+
+register_model_info(
+    ["Yuan2-2B-hf", "Yuan2-51B-hf", "Yuan2-102B-hf"],
+    "IEIYuan",
+    "https://huggingface.co/IEITYuan",
+    "Yuan2 is a Basemodel developed by IEI.",
+)

arena_elo/elo_rating/upload_battle_data.py

@@ -2,36 +2,60 @@ import fire
 import json
 import os
 import datasets
-import random
 import datetime
 from pathlib import Path
 from datetime import datetime
 from PIL import Image
 
 datasets.config.DEFAULT_MAX_BATCH_SIZE = 500
-
-def create_hf_battle_dataset(data_file: str, split="test", task_type="t2i_generation"):
-    if task_type == "t2i_generation":
-        features = datasets.Features(
+def create_hf_dataset(data_file: str, split="test"):
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=datasets.Features(
             {
-                "index": datasets.Value("int32"),
-                "tstamp": datasets.Value("int32"),
-                "prompt": datasets.Value("string"),
-                "left_model": datasets.Value("string"),
-                "left_image": datasets.Image(),
-                "right_model": datasets.Value("string"),
-                "right_image": datasets.Image(),
-                "vote_type": datasets.Value("string"),
-                "winner": datasets.Value("string"),
-                "anony": datasets.Value("bool"),
-                "judge": datasets.Value("string"),
+                "question_id": datasets.Value("string"),
+                "model": datasets.Value("string"),
+                "conversation": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
             }
-        )
-    else:
-        raise ValueError(f"Task type {task_type} not supported")
+        ),
+        split=split,
+    )
+    return hf_dataset
+
+def create_hf_battle_dataset(data_file: str, split="test"):
     hf_dataset = datasets.Dataset.from_list(
         data_file,
-        features=features,
+        features=datasets.Features(
+            {
+                "question_id": datasets.Value("string"),
+                "model_a": datasets.Value("string"),
+                "model_b": datasets.Value("string"),
+                "conversation_a": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "conversation_b": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
+                "anony": datasets.Value("bool"),
+            }
+        ),
         split=split,
     )
     return hf_dataset
@@ -57,105 +81,106 @@ def get_date_from_time_stamp(unix_timestamp: int):
 def load_battle_image(battle, log_dir):
     image_path = Path(log_dir) / f"{get_date_from_time_stamp(battle['tstamp'])}-convinput_images" / f"input_image_{battle['question_id']}.png"
     return load_image(image_path)
-
-def find_media_path(conv_id, task_type, log_dir):
-    media_directory_map = {
-        "t2i_generation": "images/generation",
-        "image_edition": "images/edition",
-        "text2video": "videos/generation"
-    }
-    if task_type == "t2i_generation":
-        media_path = Path(log_dir) / media_directory_map[task_type] / f"{conv_id}.jpg"
-    else:
-        raise ValueError(f"Task type {task_type} not supported")
-    return media_path
     
 
 def main(
-    task_type='t2i_generation',
-    # data_file: str = "./results/latest/clean_battle_conv.json",
-    data_file: str = None,
-    repo_id: str = "TIGER-Lab/GenAI-Arena-human-eval",
-    log_dir: str = os.getenv("LOGDIR", "../GenAI-Arena-hf-logs/vote_log"),
-    config_name='battle',
-    split='test',
-    token = os.environ.get("HUGGINGFACE_TOKEN", None),
-    seed=42,
+    data_file: str = "./results/latest/clean_battle_conv.json",
+    repo_id: str = "DongfuTingle/wildvision-bench",
+    log_dir: str = os.getenv("LOGDIR", "./vision-arena-logs/"),
+    mode="battle",
+    token = os.environ.get("HUGGINGFACE_TOKEN", None)
 ):
-    if data_file is None:
-        data_file = f"./results/latest/clean_battle_{task_type}.json"
-    if not os.path.exists(data_file):
-        raise ValueError(f"Data file {data_file} does not exist")
     with open(data_file, "r") as f:
         data = json.load(f)
     
-    # add index according to the tsamp
-    if seed is not None:
-        random.seed(seed)
     
     
-    data = sorted(data, key=lambda x: x['tstamp'])
-    required_keys_each_task = {
-        "image_editing": ["source_prompt", "target_prompt", "instruct_prompt"],
-        "t2i_generation": ["prompt"],
-        "video_generation": ["prompt"]
+    has_image_stats = {
+        "has_image": 0,
+        "no_image": 0,
     }
-    valid_data = []
-    for i, battle in enumerate(data):
-        if any(key not in battle['inputs'] for key in required_keys_each_task[task_type]):
-            # print(battle['inputs'])
-            # print(f"Skipping battle {i} due to missing keys")
-            continue
-        valid_data.append(battle)
-    print(f"Total battles: {len(data)}, valid battles: {len(valid_data)}, removed battles: {len(data) - len(valid_data)}")
-    data = valid_data
-    
-    # data = random.sample(data, 50 * 7+2)
-    
-    for i, battle in enumerate(data):
-        battle['index'] = i
-    
+    if mode == "keep_bad_only":
+        # anony only
+        data = [d for d in data if d["anony"]]
+        
+        new_data = []
+        for battle in data:
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
+                # we don't keep the data without image
+                continue
+            has_image_stats["has_image"] += 1
+            
+            if battle["winner"] in ["model_a", "model_b"]:
+                if battle["winner"] == "model_a":
+                    worse_model = "model_b"
+                    worse_conv = "conversation_b"
+                if battle["winner"] == "model_b":
+                    worse_model = "model_a"
+                    worse_conv = "conversation_a"
+                    
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle[worse_model],
+                    "conversation": battle[worse_conv],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+            elif battle["winner"] == "tie (bothbad)":
+                
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_a"],
+                    "conversation": battle["conversation_a"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
 
-    new_data = []
-    if task_type == 't2i_generation':
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_b"],
+                    "conversation": battle["conversation_b"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+                
+        split = "test"
+        hf_dataset = create_hf_dataset(new_data, "test")
+    
+    elif mode == "battle":
+        new_data = []
         for battle in data:
-            prompt = battle['inputs']['prompt']
-            model_a = battle['model_a']
-            model_b = battle['model_b']
-            model_a_conv_id = battle['model_a_conv_id']
-            model_b_conv_id = battle['model_b_conv_id']
-            tstamp = battle['tstamp']
-            vote_type = battle['vote_type']
-            left_image_path = find_media_path(model_a_conv_id, task_type, log_dir)
-            right_image_path = find_media_path(model_b_conv_id, task_type, log_dir)
-            left_image = load_image(left_image_path)
-            right_image = load_image(right_image_path)
-            if left_image is None or right_image is None:
-                print(f"Skipping battle {battle['index']} due to missing images")
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
                 continue
+            has_image_stats["has_image"] += 1
             new_data.append({
-                "index": battle['index'],
-                "tstamp": tstamp,
-                "prompt": prompt,
-                "left_model": model_a,
-                "left_image": left_image,
-                "right_model": model_b,
-                "right_image": right_image,
-                "vote_type": vote_type,
-                "winner": battle['winner'],
-                "anony": battle['anony'],
-                "judge": battle['judge'],
+                "question_id": battle["question_id"],
+                "model_a": battle["model_a"],
+                "model_b": battle["model_b"],
+                "conversation_a": battle["conversation_a"],
+                "conversation_b": battle["conversation_b"],
+                "language": battle["language"],
+                "image": image,
+                "turn": battle["turn"],
+                "anony": battle["anony"],
             })
         split = "test"
-        hf_dataset = create_hf_battle_dataset(new_data, split, task_type)
+        hf_dataset = create_hf_battle_dataset(new_data, "test")
     else:
-        raise ValueError(f"Task type {task_type} not supported")
+        raise ValueError(f"Invalid mode: {mode}")
 
+    print(f"Stats: {has_image_stats}")
     print(hf_dataset)
     print(f"Uploading to part {repo_id}:{split}...")
     hf_dataset.push_to_hub(
         repo_id=repo_id,
-        config_name=config_name,
+        config_name=mode,
         split=split,
         token=token,
         commit_message=f"Add vision-arena {split} dataset",

arena_elo/elo_rating/utils.py

@@ -3,20 +3,12 @@ import pytz
 import PIL
 import os 
 
-import sys
-sys.path.append('../')
-from model.model_registry import get_model_info
-
 def detect_language(text: str) -> str:
     """Detect the langauge of a string."""
-    try:
-        import polyglot  # pip3 install polyglot pyicu pycld2
-        from polyglot.detect import Detector
-        from polyglot.detect.base import logger as polyglot_logger
-        import pycld2
-    except ImportError as e:
-        print("Please install the required libraries: polyglot, pycld2: pip3 install polyglot pyicu pycld2")
-        exit(1)
+    import polyglot  # pip3 install polyglot pyicu pycld2
+    from polyglot.detect import Detector
+    from polyglot.detect.base import logger as polyglot_logger
+    import pycld2
 
     polyglot_logger.setLevel("ERROR")
 

arena_elo/generation_model_info.json

@@ -0,0 +1,42 @@
+{
+    "LCM": {
+        "Link": "https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7",
+        "License": "MIT License",
+        "Organization": "Tsinghua University"
+    },
+    "Playground v2": {
+        "Link": "https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic",
+        "License": "Playground v2 Community License",
+        "Organization": "Playground"
+    },
+    "OpenJourney": {
+        "Link": "https://huggingface.co/prompthero/openjourney",
+        "License": "creativeml-openrail-m",
+        "Organization": "PromptHero"
+    },
+    "SDXLTurbo": {
+        "Link": "https://huggingface.co/stabilityai/sdxl-turbo",
+        "License": "sai-nc-community (other)",
+        "Organization": "Stability AI"
+    },
+    "SDXL": {
+        "Link": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
+        "License": "openrail++",
+        "Organization": "Stability AI"
+    },
+    "PixArtAlpha": {
+        "Link": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS",
+        "License": "openrail++",
+        "Organization": "PixArt-alpha"
+    },
+    "SDXLLightning": {
+        "Link": "https://huggingface.co/ByteDance/SDXL-Lightning",
+        "License": "openrail++",
+        "Organization": "ByteDance"
+    },
+    "StableCascade": {
+        "Link": "https://huggingface.co/stabilityai/stable-cascade",
+        "License": "stable-cascade-nc-community (other)",
+        "Organization": "Stability AI"
+    }
+}

arena_elo/results/20240315/elo_results_image_editing.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5cef00c45d392a30913b367825270fcee5fd29e5c830866eef3d07146b3502f3
-size 57091
+oid sha256:e528d30840c8a5787b0d2f08f27758b02f7eb718ccab695010b30df2127efe5e
+size 57064

arena_elo/results/20240327/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f525abe69feb822d341929b27ef7660ddd5e6ff0491bed8383a8e3d19f0342bd
-size 62414
+oid sha256:fec01fe5af62dce3990634cffd1d926330ccbf170ef0c3b5d2f07fb06c4cf149
+size 65189

arena_elo/results/20240327/t2i_generation_leaderboard.csv

@@ -1,10 +1,11 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Playground v2.5,Playground v2.5,1226.2872445351936,1246.1685934024742,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1105.3322734027522,1087.9198960927265,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1091.4371447234744,1090.676108819673,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1043.235902888147,1045.0529259890538,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6412075829058,1006.9966036187151,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,964.7626495363717,969.5241392802999,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2113859675355,914.3805456579931,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.2224045541894,832.2282703082603,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8697868094328,810.2118373597045,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
+Playground v2.5,Playground v2.5,1212.4660228554317,1233.021110469063,N/A,N/A,N/A
+StableCascade,StableCascade,1098.8180832734447,1081.4707812969855,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+PlayGroundV2,PlayGroundV2,1089.993871580802,1088.6262085724481,N/A,N/A,N/A
+Playground v2,Playground v2,1049.6156124554975,1051.618375116693,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+SDXLLightning,SDXLLightning,1036.8582186059539,1039.3079223370821,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1016.2085497703334,1002.5100184720693,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL,SDXL,960.5073412035289,965.3037978455568,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,907.997473382927,910.1644152252661,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+OpenJourney,OpenJourney,836.9689192463355,827.9470053715127,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,790.5659076257482,805.8155782210948,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240328/clean_battle_image_editing.json

@@ -1,890 +0,0 @@
-[
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712630.872
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712699.668
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712896.0427
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712929.7061
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713147.0445
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713198.9284
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713210.1306
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713747.5115
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707715613.7226
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765708.2644
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765861.2742
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707765975.0206
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707768866.9065
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707771673.2989
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784377.6617
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784466.8915
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784983.9581
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707785277.16
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795299.0619
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795798.752
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707796435.7996
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797278.7369
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797279.6004
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805086.9739
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805220.3253
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805332.6322
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805476.0509
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707818374.3438
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834631.9088
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834954.0147
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835366.544
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835643.6178
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835789.25
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707836852.671
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707836952.6082
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707837020.7148
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707837226.2259
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838166.1449
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838405.0013
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839133.3126
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839484.6824
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707850104.2499
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851384.7689
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851936.9466
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707852836.3291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707852878.673
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707853008.1359
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707856807.6229
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707863740.3507
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707866312.1118
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883083.3533
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883181.1397
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883187.9173
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883507.587
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883939.6125
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707892689.4407
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707908988.749
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707912639.2701
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707917685.9574
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707919429.336
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932651.9192
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932749.3107
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707933208.5797
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707945335.6341
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708031168.6838
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708038931.5388
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708057382.78
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093689.8237
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093910.4683
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095090.8232
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095305.4665
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708140553.1694
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145512.3656
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145724.4127
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708146846.5098
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708189738.4864
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708235874.9246
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708257619.7115
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708341265.7655
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708350183.3086
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708399707.1681
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441502.4707
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441716.8195
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546759.2009
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546805.4892
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708547082.7124
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547166.9685
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547293.7107
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708575046.0529
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708615466.9264
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708615516.3341
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205399.0098
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205767.8923
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709443700.05
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709702898.9291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710091925.1861
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710517781.1525
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1710517859.2942
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710535672.9791
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1711610477.1213
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711629129.3894
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711629705.2246
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1711630362.5575
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": false,
-    "tstamp": 1711631112.5207
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1711631690.5127
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": false,
-    "tstamp": 1711633200.2923
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": false,
-    "tstamp": 1711633594.9922
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711635443.3071
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1711635899.3088
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1711639015.428
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": false,
-    "tstamp": 1711646372.1201
-  }
-]

arena_elo/results/20240328/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1430e6703dd6fc1e5b8ce06b11bb3a47516763a33edaf99e4c8547da5d9a8516
-size 57064

arena_elo/results/20240328/image_editing_leaderboard.csv

@@ -1,8 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Prompt2prompt,Prompt2prompt,1227.5508595026165,1158.5510681980204,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-InstructPix2Pix,InstructPix2Pix,1160.2057367236093,1071.0628993075604,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
-PNP,PNP,1142.693603173293,1165.4957550490212,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-MagicBrush,MagicBrush,1053.1728944865915,1130.5422054860635,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-Pix2PixZero,Pix2PixZero,918.6047552604578,960.3217617445996,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
-CycleDiffusion,CycleDiffusion,865.0529105743963,813.4794423328381,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
-SDEdit,SDEdit,632.7192402790356,700.546867881897,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240330/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e963f9d4b66d29c2f05a3923eff56cebd1f09b07223ac069456e08dc6143cda8
-size 66894

arena_elo/results/20240330/t2i_generation_leaderboard.csv

@@ -1,10 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Playground v2.5,Playground v2.5,1226.2872445351936,1236.5076527218755,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1105.3322734027522,1062.0980902577003,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1091.4371447234744,1087.3576445526567,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1043.235902888147,1019.4526672266176,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6412075829058,1001.5090282446616,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,964.7626495363717,969.8928133531979,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2113859675355,914.9478831930971,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.2224045541894,835.4563491411935,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8697868094328,812.962889153237,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240408/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dd88783d1cf752a0977152f7e16e88b54759173cbb04fb55e9392703ff4819f5
-size 66931

arena_elo/results/20240408/t2i_generation_leaderboard.csv

@@ -1,10 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Playground v2.5,Playground v2.5,1226.2872445351936,1233.8616648345985,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1105.3322734027522,1031.1844458387527,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1091.4371447234744,1093.6921447327898,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1043.235902888147,1004.2360415152086,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6412075829058,999.6264863931511,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,964.7626495363717,975.3460583905047,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2113859675355,927.1873122981513,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.2224045541894,848.6657236271969,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8697868094328,828.5108951096241,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240411/clean_battle_image_editing.json

@@ -1,906 +0,0 @@
-[
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712630.872
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712699.668
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712896.0427
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712929.7061
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713147.0445
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713198.9284
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713210.1306
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713747.5115
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707715613.7226
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765708.2644
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765861.2742
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707765975.0206
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707768866.9065
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707771673.2989
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784377.6617
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784466.8915
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784983.9581
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707785277.16
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795299.0619
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795798.752
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707796435.7996
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797278.7369
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797279.6004
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805086.9739
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805220.3253
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805332.6322
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805476.0509
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707818374.3438
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834631.9088
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834954.0147
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835366.544
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835643.6178
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835789.25
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707836852.671
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707836952.6082
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707837020.7148
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707837226.2259
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838166.1449
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838405.0013
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839133.3126
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839484.6824
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707850104.2499
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851384.7689
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851936.9466
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707852836.3291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707852878.673
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707853008.1359
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707856807.6229
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707863740.3507
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707866312.1118
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883083.3533
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883181.1397
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883187.9173
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883507.587
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883939.6125
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707892689.4407
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707908988.749
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707912639.2701
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707917685.9574
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707919429.336
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932651.9192
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932749.3107
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707933208.5797
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707945335.6341
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708031168.6838
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708038931.5388
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708057382.78
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093689.8237
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093910.4683
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095090.8232
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095305.4665
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708140553.1694
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145512.3656
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145724.4127
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708146846.5098
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708189738.4864
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708235874.9246
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708257619.7115
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708341265.7655
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708350183.3086
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708399707.1681
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441502.4707
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441716.8195
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546759.2009
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546805.4892
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708547082.7124
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547166.9685
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547293.7107
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708575046.0529
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708615466.9264
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708615516.3341
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205399.0098
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205767.8923
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709443700.05
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709702898.9291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710091925.1861
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710517781.1525
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1710517859.2942
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710535672.9791
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1711610477.1213
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711629129.3894
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711629705.2246
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Pix2PixZero",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1711630362.5575
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": true,
-    "tstamp": 1711631112.5207
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1711631690.5127
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": true,
-    "tstamp": 1711633200.2923
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_127.0.0.1",
-    "anony": true,
-    "tstamp": 1711633594.9922
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711635443.3071
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1711635899.3088
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1711639015.428
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.7.189",
-    "anony": true,
-    "tstamp": 1711646372.1201
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1712873850.0636
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1712876598.7667
-  }
-]

arena_elo/results/20240411/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d66a54af51d2ecf89f461dbb4e15090d084638596952d3541ce369798a525ff3
-size 57096

arena_elo/results/20240411/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:caf98f560387fa9d6b8c233e9915807adad62315cfdd6d4a5e7c9fda30140eb8
-size 62422

arena_elo/results/20240411/image_editing_leaderboard.csv

@@ -1,8 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Prompt2prompt,Prompt2prompt,1188.219371435949,1160.9021011448333,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-PNP,PNP,1133.8594830307645,1160.2784411172045,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-InstructPix2Pix,InstructPix2Pix,1086.6617653998492,1065.4343032662,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
-MagicBrush,MagicBrush,1084.8708678670623,1120.3917913590851,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-Pix2PixZero,Pix2PixZero,983.9050014855375,949.5286840298457,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
-CycleDiffusion,CycleDiffusion,847.634435323394,811.6166545238106,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
-SDEdit,SDEdit,674.8490754574439,731.8480245590208,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240411/t2i_generation_leaderboard.csv

@@ -1,10 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGround V2,PlayGround V2,1096.7894880225679,1099.8051043857877,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-PlayGround V2.5,PlayGround V2.5,1087.8676967844767,1102.012177335679,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1055.9173326915914,1059.3764815279687,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-PixArtAlpha,PixArtAlpha,1033.9990481857885,1022.7034421485712,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXLLightning,SDXLLightning,1033.7993884424232,1038.4887196068619,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-SDXL,SDXL,1001.9345229118052,1000.9893451213411,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,954.8868434684313,951.3491425503697,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,888.3709717134242,873.7483257587076,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,846.4347077794937,852.2372365264126,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240428/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c1b4f1daab3429c7656eb8b3b2128a127480fa8212b17a1a98207884d7ce7a9f
-size 58442

arena_elo/results/20240428/image_editing_leaderboard.csv

@@ -1,8 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-Prompt2prompt,Prompt2prompt,1224.5951620965877,1133.887231157847,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-InstructPix2Pix,InstructPix2Pix,1162.3591990023222,1059.7394666236296,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
-PNP,PNP,1142.872221219748,1117.461082043853,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-MagicBrush,MagicBrush,1053.6353139288728,1055.6074426532264,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-Pix2PixZero,Pix2PixZero,918.4266240422415,853.535635519584,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
-CycleDiffusion,CycleDiffusion,865.2495984976465,775.6226309361784,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
-SDEdit,SDEdit,632.8618812125814,680.2047869803968,MIT License,Stanford University,https://sde-image-editing.github.io

arena_elo/results/20240501/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9b79d128ba01570bc59c5f48e1c0640f2541817ce1a77abb3e16131884288b1a
-size 65313

arena_elo/results/20240501/t2i_generation_leaderboard.csv

@@ -1,11 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGround V2.5,PlayGround V2.5,1157.785440865029,1197.7936802344343,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-StableCascade,StableCascade,1116.6696847615349,1116.9442071854512,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-PlayGround V2,PlayGround V2,1110.1291971452683,1120.6591618464581,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-PixArtAlpha,PixArtAlpha,1042.1316579959862,1040.3305680293547,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXLLightning,SDXLLightning,1036.0784815928241,1056.600050803737,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-SDXL,SDXL,987.5686859787551,1003.0595102032345,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-PixArtSigma,PixArtSigma,948.0067582557859,961.4040676622378,N/A,N/A,N/A
-SDXLTurbo,SDXLTurbo,931.094996526404,945.5610964234802,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,855.7449360962327,860.1159058283633,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,814.7901607821794,840.5627577743975,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240516/clean_battle_image_editing.json

@@ -1,1578 +0,0 @@
-[
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712630.872
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712699.668
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707712896.0427
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707712929.7061
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713147.0445
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713198.9284
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713210.1306
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707713747.5115
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707715613.7226
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765708.2644
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707765861.2742
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707765975.0206
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707768866.9065
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707771673.2989
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784377.6617
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784466.8915
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707784983.9581
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707785277.16
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795299.0619
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707795798.752
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707796435.7996
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797278.7369
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707797279.6004
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805086.9739
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805220.3253
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805332.6322
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707805476.0509
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707818374.3438
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834631.9088
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707834954.0147
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835366.544
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835643.6178
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707835789.25
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707836852.671
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707836952.6082
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707837020.7148
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707837226.2259
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838166.1449
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707838405.0013
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839133.3126
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707839484.6824
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707850104.2499
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851384.7689
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707851936.9466
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707852836.3291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707852878.673
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707853008.1359
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707856807.6229
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707863740.3507
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707866312.1118
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883083.3533
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883181.1397
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883187.9173
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883507.587
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707883939.6125
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707892689.4407
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707908988.749
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707912639.2701
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707917685.9574
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707919429.336
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932651.9192
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707932749.3107
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1707933208.5797
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1707945335.6341
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708031168.6838
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708038931.5388
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708057382.78
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093689.8237
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708093910.4683
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095090.8232
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708095305.4665
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708140553.1694
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145512.3656
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708145724.4127
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708146846.5098
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708189738.4864
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708235874.9246
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708257619.7115
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708341265.7655
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708350183.3086
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708399707.1681
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441502.4707
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708441716.8195
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546759.2009
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708546805.4892
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708547082.7124
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547166.9685
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708547293.7107
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708575046.0529
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1708615466.9264
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1708615516.3341
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205399.0098
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1709205767.8923
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709443700.05
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1709702898.9291
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710091925.1861
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710517781.1525
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": false,
-    "tstamp": 1710517859.2942
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_::1",
-    "anony": true,
-    "tstamp": 1710535672.9791
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1714359818.6646
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1714363016.9972
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "CosXLEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1714715956.3416
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": false,
-    "tstamp": 1714759928.3804
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715246275.0118
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CosXLEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715247590.2235
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1715406266.2562
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1715406354.5284
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.2.201",
-    "anony": false,
-    "tstamp": 1715406371.8227
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1715406418.5066
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": false,
-    "tstamp": 1715406449.9401
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1715406466.5778
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715620708.6361
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CosXLEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": false,
-    "tstamp": 1715621013.5373
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715661224.0507
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715661259.6143
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715661288.6018
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715661310.3621
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715718742.1258
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715718773.1054
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CosXLEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715718785.2832
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715718804.143
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715718826.0248
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715718869.0041
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715718904.9307
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715718933.1272
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715718954.8497
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715718966.8633
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719000.6673
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719019.5495
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719035.903
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719046.925
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719059.6291
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715719076.6727
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719086.7836
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719109.8071
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719122.8237
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715719134.1345
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715719153.4359
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715719160.5285
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InstructPix2Pix",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715719171.4473
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719184.6227
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719210.0429
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719219.6447
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719237.7036
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719249.4321
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719257.5877
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719273.7637
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715719288.4629
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719299.1712
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719306.5928
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715719356.0694
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CosXLEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719368.0491
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "CycleDiffusion",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719379.185
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719389.0771
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715719397.7162
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719406.4165
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "PNP",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719429.1002
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719435.4694
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719454.4526
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715719470.154
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719482.3114
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "PNP",
-    "winner": "tie",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719499.9643
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719513.7317
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "PNP",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715719527.69
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719542.751
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InfEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715719560.9912
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719575.3291
-  },
-  {
-    "model_a": "PNP",
-    "model_b": "CosXLEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715719581.9552
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719591.9907
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719601.8819
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719612.1837
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719620.469
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "MagicBrush",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719627.34
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719632.694
-  },
-  {
-    "model_a": "Prompt2prompt",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719652.2038
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719661.8855
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719677.2949
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719687.3022
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719699.47
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "InfEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719706.2375
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715719717.3564
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "CosXLEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715719722.5542
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "InstructPix2Pix",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715719728.5417
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715719737.2385
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715815138.5243
-  },
-  {
-    "model_a": "CosXLEdit",
-    "model_b": "Prompt2prompt",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715815152.0033
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "Prompt2prompt",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715815169.0475
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "SDEdit",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715815187.1917
-  },
-  {
-    "model_a": "InstructPix2Pix",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715815197.5233
-  },
-  {
-    "model_a": "Pix2PixZero",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715815209.8285
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "MagicBrush",
-    "winner": "model_b",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715815228.6736
-  },
-  {
-    "model_a": "InfEdit",
-    "model_b": "Pix2PixZero",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.2.201",
-    "anony": true,
-    "tstamp": 1715815236.3935
-  },
-  {
-    "model_a": "SDEdit",
-    "model_b": "PNP",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.25.191",
-    "anony": true,
-    "tstamp": 1715815265.9705
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "SDEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715815278.5019
-  },
-  {
-    "model_a": "CycleDiffusion",
-    "model_b": "CosXLEdit",
-    "winner": "tie (bothbad)",
-    "judge": "arena_user_10.16.15.199",
-    "anony": true,
-    "tstamp": 1715815294.5978
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "InfEdit",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.17.217",
-    "anony": true,
-    "tstamp": 1715815325.4468
-  },
-  {
-    "model_a": "MagicBrush",
-    "model_b": "Pix2PixZero",
-    "winner": "model_a",
-    "judge": "arena_user_10.16.41.118",
-    "anony": true,
-    "tstamp": 1715913098.6617
-  }
-]

arena_elo/results/20240516/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:837f11fd6cda1fe2d6a5cc1c239a207725ad0157b16282303cb684427ddc7e9d
-size 62484

arena_elo/results/20240516/image_editing_leaderboard.csv

@@ -1,10 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-CosXLEdit,CosXLEdit,1097.63559213644,1085.7285800995926,cosxl-nc-community,Stability AI,https://huggingface.co/spaces/multimodalart/cosxl
-MagicBrush,MagicBrush,1075.1489922450316,1086.8819832924794,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-InfEdit,InfEdit,1065.4719519196174,1090.684638162955,Apache-2.0,"University of Michigan, University of California, Berkeley",https://huggingface.co/spaces/sled-umich/InfEdit
-Prompt2prompt,Prompt2prompt,1063.1432047252297,1060.8146250689238,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-InstructPix2Pix,InstructPix2Pix,1043.9312648233226,1028.7932718869638,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
-PNP,PNP,1022.4342554377677,1043.322342347598,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-Pix2PixZero,Pix2PixZero,891.2979039265506,886.7359371585381,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
-SDEdit,SDEdit,890.443823405714,880.5508125882768,MIT License,Stanford University,https://sde-image-editing.github.io
-CycleDiffusion,CycleDiffusion,850.4930113803264,836.4878093946726,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion

arena_elo/results/20240517/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:93808a9ce2f497109d0fc708e4055b6463a692502ef541ff28352f52b612916d
-size 68172

arena_elo/results/20240517/t2i_generation_leaderboard.csv

@@ -1,12 +0,0 @@
-key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGround V2.5,PlayGround V2.5,1136.9514432133128,1081.5838551712898,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-PlayGround V2,PlayGround V2,1099.4286233187172,1042.590911846903,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1062.4565867132737,1004.4096880141087,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-StableCascade,StableCascade,1061.93020315328,1006.1117357811837,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-PixArtAlpha,PixArtAlpha,1051.847602698194,981.1247821885942,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-PixArtSigma,PixArtSigma,1049.8339911951734,989.7640320919886,openrail++,PixArt-alpha,https://fal.ai/models/fal-ai/pixart-sigma
-SDXL,SDXL,999.6167439144875,941.9623909945509,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,933.468824554199,875.8124778188443,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-LCM(v1.5/XL),LCM(v1.5/XL),929.425577747465,865.7356218313212,openrail++,Latent Consistency,https://fal.ai/models/fal-ai/fast-lcm-diffusion/api
-OpenJourney,OpenJourney,857.2709081764949,793.4952273226107,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,817.7694953154022,773.4948395309905,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240525/elo_results_image_editing.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a90694074e1b68a62bd75cdf0c81eb545dfcc115da34e9efdb215d668bd13196
-size 62502

arena_elo/results/20240525/elo_results_t2i_generation.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7172486b8454e25f9b3a9df84e55d2dcce923a3b63e091fd8d165b63bbde7bc4
-size 68170