Update app.py

app.py

@@ -29,7 +29,11 @@ def make_arena_leaderboard_md(arena_df):
     total_models = len(arena_df)
     space = "   "
     leaderboard_md = f"""
-Total #models: **{total_models}**.{space} Total #votes: **{"{:,}".format(total_votes)}**.{space} Last updated: June 1, 2024.
+Three benchmarks are displayed: **Test Task 1**, **Test Task 2**, **Test Task 3**.
+
+Higher values are better for all benchmarks.
+
+Total #models: **{total_models}**.{space} Last updated: June 1, 2024.
 
 """
     return leaderboard_md
@@ -45,14 +49,6 @@ def make_category_arena_leaderboard_md(arena_df, arena_subset_df, name="Overall"
 """
     return leaderboard_md
 
-def make_full_leaderboard_md(elo_results):
-    leaderboard_md = f"""
-Three benchmarks are displayed: **Test Task 1**, **Test Task 2**, **Test Task 3**.
-
-Higher values are better for all benchmarks.
-"""
-    return leaderboard_md
-
 
 def make_leaderboard_md_live(elo_results):
     leaderboard_md = f"""
@@ -96,25 +92,11 @@ def update_elo_components(max_num_files, elo_results_file):
     basic_component_values[5] = md4
 
 
-def update_worker(max_num_files, interval, elo_results_file):
-    while True:
-        tic = time.time()
-        update_elo_components(max_num_files, elo_results_file)
-        durtaion = time.time() - tic
-        print(f"update duration: {durtaion:.2f} s")
-        time.sleep(max(interval - durtaion, 0))
-
-
-def load_demo(url_params, request: gr.Request):
-    logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}")
-    return basic_component_values + leader_component_values
-
-
 def model_hyperlink(model_name, link):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 
-def load_leaderboard_table_csv(filename, add_hyperlink=True):
+def load_leaderboard_table_csv(filename, add_hyperlink=False):
     lines = open(filename).readlines()
     heads = [v.strip() for v in lines[0].split(",")]
     rows = []
@@ -180,9 +162,7 @@ def get_full_table(model_table_df):
         row.append(model_name)
         row.append(np.nan)
         row.append(np.nan)
-        row.append(np.nan)
-        # row.append(model_table_df.iloc[i]["MT-bench (score)"])
-        # row.append(model_table_df.iloc[i]["MMLU"])
+        row.append(np.nan)\
         # Organization
         row.append(model_table_df.iloc[i]["Organization"])
         # license
@@ -192,86 +172,6 @@ def get_full_table(model_table_df):
     values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9)
     return values
 
-def create_ranking_str(ranking, ranking_difference):
-    if ranking_difference > 0:
-        # return f"{int(ranking)} (\u2191{int(ranking_difference)})"
-        return f"{int(ranking)} \u2191"
-    elif ranking_difference < 0:
-        # return f"{int(ranking)} (\u2193{int(-ranking_difference)})"
-        return f"{int(ranking)} \u2193"
-    else:
-        return f"{int(ranking)}"
-    
-def recompute_final_ranking(arena_df):
-    # compute ranking based on CI
-    ranking = {}
-    for i, model_a in enumerate(arena_df.index):
-        ranking[model_a] = 1
-        for j, model_b in enumerate(arena_df.index):
-            if i == j:
-                continue
-            if arena_df.loc[model_b]["rating_q025"] > arena_df.loc[model_a]["rating_q975"]:
-                ranking[model_a] += 1
-    return list(ranking.values())
-    
-def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
-    arena_df = arena_df.sort_values(by=["final_ranking", "rating"], ascending=[True, False])
-    arena_df["final_ranking"] = recompute_final_ranking(arena_df)
-    arena_df = arena_df.sort_values(by=["final_ranking"], ascending=True)
-
-    # arena_df["final_ranking"] = range(1, len(arena_df) + 1)
-    # sort by rating
-    if arena_subset_df is not None:
-        # filter out models not in the arena_df
-        arena_subset_df = arena_subset_df[arena_subset_df.index.isin(arena_df.index)]
-        arena_subset_df = arena_subset_df.sort_values(by=["rating"], ascending=False)
-        # arena_subset_df = arena_subset_df.sort_values(by=["final_ranking"], ascending=True)
-        arena_subset_df["final_ranking"] = recompute_final_ranking(arena_subset_df)
-        # keep only the models in the subset in arena_df and recompute final_ranking
-        arena_df = arena_df[arena_df.index.isin(arena_subset_df.index)]
-        # recompute final ranking
-        arena_df["final_ranking"] = recompute_final_ranking(arena_df)
-
-        # assign ranking by the order
-        arena_subset_df["final_ranking_no_tie"] = range(1, len(arena_subset_df) + 1)
-        arena_df["final_ranking_no_tie"] = range(1, len(arena_df) + 1)
-        # join arena_df and arena_subset_df on index
-        arena_df = arena_subset_df.join(arena_df["final_ranking"], rsuffix="_global", how="inner")
-        arena_df["ranking_difference"] =  arena_df["final_ranking_global"] - arena_df["final_ranking"]
-
-        arena_df = arena_df.sort_values(by=["final_ranking", "rating"], ascending=[True, False])
-        arena_df["final_ranking"] = arena_df.apply(lambda x: create_ranking_str(x["final_ranking"], x["ranking_difference"]), axis=1)
-
-    values = []
-    for i in range(len(arena_df)):
-        row = []
-        model_key = arena_df.index[i]
-        try: # this is a janky fix for where the model key is not in the model table (model table and arena table dont contain all the same models)
-            model_name = model_table_df[model_table_df["key"] == model_key]["Model"].values[
-                0
-            ]
-            # rank
-            ranking = arena_df.iloc[i].get("final_ranking") or i+1
-            row.append(ranking)
-            if arena_subset_df is not None:
-                row.append(arena_df.iloc[i].get("ranking_difference") or 0)
-            # model display name
-            row.append(model_name)
-            # elo rating
-            row.append(round(arena_df.iloc[i]["rating"]))
-            # Organization
-            row.append(
-                model_table_df[model_table_df["key"] == model_key]["Organization"].values[0]
-            )
-            # license
-            row.append(
-                model_table_df[model_table_df["key"] == model_key]["License"].values[0]
-            )
-            values.append(row)
-        except Exception as e:
-            print(f"{model_key} - {e}")
-    return values
-
 key_to_category_name = {
     "full": "Overall",
 }
@@ -304,9 +204,8 @@ def build_leaderboard_tab(results_file, leaderboard_table_file, show_plot=False)
         model_table_df = pd.DataFrame(data)
 
         with gr.Tabs() as tabs:
-            # arena table
             arena_table_vals = get_full_table(model_table_df) 
-            with gr.Tab("Arena Elo", id=0):
+            with gr.Tab("Full leaderboard", id=0):
                 md = make_arena_leaderboard_md(arena_df)
                 leaderboard_markdown = gr.Markdown(md, elem_id="leaderboard_markdown")
                 with gr.Row():
@@ -350,40 +249,6 @@ def build_leaderboard_tab(results_file, leaderboard_table_file, show_plot=False)
 
                 leader_component_values[:] = [default_md]
 
-            # with gr.Tab("Full Leaderboard", id=0):
-            #     md = make_full_leaderboard_md(elo_results)
-            #     gr.Markdown(md, elem_id="leaderboard_markdown")
-            #     with gr.Row():
-            #         with gr.Column(scale=2):
-            #             category_dropdown = gr.Dropdown(choices=list(arena_dfs.keys()), label="Category", value="Overall")
-            #         default_category_details = make_category_arena_leaderboard_md(arena_df, arena_df, name="Overall")
-            #         with gr.Column(scale=4, variant="panel"):
-            #             category_deets = gr.Markdown(default_category_details, elem_id="category_deets")
-
-            #     full_table_vals = get_full_table(model_table_df)
-            #     display_df = gr.Dataframe(
-            #         headers=[
-            #             "🤖 Model",
-            #             "⭐ Task 1",
-            #             "📈 Task 2",
-            #             "📚 Task 3",
-            #             "Organization",
-            #             "License",
-            #         ],
-            #         datatype=["markdown", "number", "number", "number", "str", "str"],
-            #         value=full_table_vals,
-            #         elem_id="full_leaderboard_dataframe",
-            #         column_widths=[200, 100, 100, 100, 150, 150],
-            #         height=700,
-            #         wrap=True,
-            #     )
-            #     gr.Markdown(
-            #         f"""Note: .
-            #         """,
-            #         elem_id="leaderboard_markdown"
-            #     )
-
-            #     leader_component_values[:] = [default_md]
         if not show_plot:
             gr.Markdown(
                 """ ## Submit your model [here]().
@@ -394,7 +259,7 @@ def build_leaderboard_tab(results_file, leaderboard_table_file, show_plot=False)
         pass
 
     def update_leaderboard_df(arena_table_vals):
-        elo_datarame = pd.DataFrame(arena_table_vals, columns=[ "Rank", "🤖 Model", "⭐ Arena Elo", "Organization", "License"])
+        elo_datarame = pd.DataFrame(arena_table_vals, columns=["Rank", "🤖 Model", "⭐ Task 1", "📈 Task 2", "📚 Task 3", "Organization", "License"])
 
         # goal: color the rows based on the rank with styler
         def highlight_max(s):
@@ -414,51 +279,31 @@ def build_leaderboard_tab(results_file, leaderboard_table_file, show_plot=False)
         arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df = arena_subset_df if category != "Overall" else None)
         if category != "Overall":
             arena_values = update_leaderboard_df(arena_values)
-            arena_values = gr.Dataframe(
-                    headers=[
-                        "Rank",
-                        "🤖 Model",
-                        "⭐ Arena Elo",
-                        "Organization",
-                        "License",
-                    ],
-                    datatype=[
-                        "number",
-                        "markdown",
-                        "number",
-                        "str",
-                        "str",
-                    ],
-                    value=arena_values,
-                    elem_id="arena_leaderboard_dataframe",
-                    height=700,
-                    column_widths=[60, 190, 110, 160, 150, 140],
-                    wrap=True,
-                )
-        else:
-            arena_values = gr.Dataframe(
-                    headers=[
-                        "Rank",
-                        "🤖 Model",
-                        "⭐ Arena Elo",
-                        "Organization",
-                        "License",
-                    ],
-                    datatype=[
-                        "number",
-                        "markdown",
-                        "number",
-                        "str",
-                        "str",
-                    ],
-                    value=arena_values,
-                    elem_id="arena_leaderboard_dataframe",
-                    height=700,
-                    column_widths=[70, 190, 110, 160, 150, 140],
-                    wrap=True,
-                )
-
-
+        arena_values = gr.Dataframe(
+                headers=[
+                    "Rank",
+                    "🤖 Model",
+                    "⭐ Task 1",
+                    "📈 Task 2",
+                    "📚 Task 3",
+                    "Organization",
+                    "License",
+                ],
+                datatype=[
+                    "number",
+                    "markdown",
+                    "number",
+                    "number",
+                    "number",
+                    "str",
+                    "str",
+                ],
+                value=arena_values,
+                elem_id="arena_leaderboard_dataframe",
+                height=700,
+                column_widths=[70, 190, 110, 110, 110, 150, 140],
+                wrap=True,
+            )
         leaderboard_md = make_category_arena_leaderboard_md(arena_df, arena_subset_df, name=category)
         return arena_values, leaderboard_md