add filter: Use Case Area

app.py

@@ -34,6 +34,7 @@ def update_table(
     llm_query: list,
     llm_provider_query: list,
     accuracy_method_query: str,
+    use_case_area_query: list,
     use_case_query: list,
     use_case_type_query: list,
     # type_query: list,
@@ -49,20 +50,49 @@ def update_table(
     filtered_df = filter_accuracy_method_func(filtered_df, accuracy_method_query)
 
     filtered_df["Use Case Area"] = filtered_df["Use Case Name"].apply(lambda x: x.split(": ")[0])
-    # print(filtered_df["Use Case Area"].unique())
+    filtered_df = filter_use_case_area_func(filtered_df, use_case_area_query)
     filtered_df = filter_use_case_func(filtered_df, use_case_query)
     filtered_df = filter_use_case_type_func(filtered_df, use_case_type_query)
     df = select_columns(filtered_df, columns)
     return df
 
 
+def init_leaderboard_df(
+    leaderboard_df: pd.DataFrame,
+    columns: list,
+    llm_query: list,
+    llm_provider_query: list,
+    accuracy_method_query: str,
+    use_case_area_query: list,
+    use_case_query: list,
+    use_case_type_query: list,
+):
+    return update_table(
+        leaderboard_df,
+        columns,
+        llm_query,
+        llm_provider_query,
+        accuracy_method_query,
+        use_case_area_query,
+        use_case_query,
+        use_case_type_query,
+    )
+
+
 def filter_accuracy_method_func(df: pd.DataFrame, accuracy_method_query: str) -> pd.DataFrame:
     return df[df["Accuracy Method"] == accuracy_method_query]
 
 
+def filter_use_case_area_func(df: pd.DataFrame, use_case_area_query: list) -> pd.DataFrame:
+    return df[
+        df["Use Case Area"].apply(
+            lambda x: len(set([_.strip() for _ in x.split("&")]).intersection(use_case_area_query))
+        )
+        > 0
+    ]
+
+
 def filter_use_case_func(df: pd.DataFrame, use_case_query: list) -> pd.DataFrame:
-    # print(use_case_query)
-    # print(df[df["Use Case Name"].isin(["Service: Conversation summary"])])
     return df[df["Use Case Name"].isin(use_case_query)]
 
 
@@ -170,7 +200,33 @@ with demo:
                 #     )
             with gr.Row():
                 with gr.Column():
-                    filter_use_case_type = gr.CheckboxGroup(
+                    filter_llm = gr.CheckboxGroup(
+                        choices=list(original_df["Model Name"].unique()),
+                        value=list(original_df["Model Name"].unique()),
+                        label="Model Name",
+                        info="",
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_llm_provider = gr.CheckboxGroup(
+                        choices=list(original_df["LLM Provider"].unique()),
+                        value=list(original_df["LLM Provider"].unique()),
+                        label="LLM Provider",
+                        info="",
+                        interactive=True,
+                    )
+            with gr.Row():
+                filter_use_case = gr.CheckboxGroup(
+                    choices=list(original_df["Use Case Name"].unique()),
+                    value=list(original_df["Use Case Name"].unique()),
+                    label="Use Case",
+                    info="",
+                    # multiselect=True,
+                    interactive=True,
+                )
+            with gr.Row():
+                with gr.Column():
+                    filter_use_case_area = gr.CheckboxGroup(
                         choices=["Service", "Sales"],
                         value=["Service", "Sales"],
                         label="Use Case Area",
@@ -185,15 +241,15 @@ with demo:
                         info="",
                         interactive=True,
                     )
-                with gr.Column():
-                    filter_use_case = gr.Dropdown(
-                        choices=list(original_df["Use Case Name"].unique()),
-                        value=list(original_df["Use Case Name"].unique()),
-                        label="Use Case",
-                        info="",
-                        multiselect=True,
-                        interactive=True,
-                    )
+                # with gr.Column():
+                #     filter_use_case = gr.Dropdown(
+                #         choices=list(original_df["Use Case Name"].unique()),
+                #         value=list(original_df["Use Case Name"].unique()),
+                #         label="Use Case",
+                #         info="",
+                #         multiselect=True,
+                #         interactive=True,
+                #     )
                 with gr.Column():
                     filter_metric_area = gr.CheckboxGroup(
                         choices=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
@@ -217,25 +273,35 @@ with demo:
                         info="Range: 0.0 to 4.0",
                         interactive=True,
                     )
-                with gr.Column():
-                    filter_llm = gr.CheckboxGroup(
-                        choices=list(original_df["Model Name"].unique()),
-                        value=list(leaderboard_df["Model Name"].unique()),
-                        label="Model Name",
-                        info="",
-                        interactive=True,
-                    )
-                with gr.Column():
-                    filter_llm_provider = gr.CheckboxGroup(
-                        choices=list(original_df["LLM Provider"].unique()),
-                        value=list(leaderboard_df["LLM Provider"].unique()),
-                        label="LLM Provider",
-                        info="",
-                        interactive=True,
-                    )
+                # with gr.Column():
+                #     filter_llm = gr.CheckboxGroup(
+                #         choices=list(original_df["Model Name"].unique()),
+                #         value=list(leaderboard_df["Model Name"].unique()),
+                #         label="Model Name",
+                #         info="",
+                #         interactive=True,
+                #     )
+                # with gr.Column():
+                #     filter_llm_provider = gr.CheckboxGroup(
+                #         choices=list(original_df["LLM Provider"].unique()),
+                #         value=list(leaderboard_df["LLM Provider"].unique()),
+                #         label="LLM Provider",
+                #         info="",
+                #         interactive=True,
+                #     )
 
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                # value=leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                value=init_leaderboard_df(
+                    leaderboard_df,
+                    shown_columns.value,
+                    filter_llm.value,
+                    filter_llm_provider.value,
+                    filter_accuracy_method.value,
+                    filter_use_case_area.value,
+                    filter_use_case.value,
+                    filter_use_case_type.value,
+                ),
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
@@ -268,6 +334,7 @@ with demo:
                 filter_llm,
                 filter_llm_provider,
                 filter_accuracy_method,
+                filter_use_case_area,
                 filter_use_case,
                 filter_use_case_type,
                 # filter_columns_type,
@@ -283,6 +350,7 @@ with demo:
                         filter_llm,
                         filter_llm_provider,
                         filter_accuracy_method,
+                        filter_use_case_area,
                         filter_use_case,
                         filter_use_case_type,
                         # filter_columns_type,

crm-results/hf_leaderboard_latency_cost.csv

@@ -0,0 +1,37 @@
+Model Name,Use Case Type (Long vs Short),Platform,Mean Latency (sec) per Request,Mean Output Tokens,Mean Cost per 1K Requests,Cost Band
+AI21 Jamba-Instruct,Long,AI21,4.0,232.9,1.6,High
+AI21 Jamba-Instruct,Short,AI21,4.0,243.9,0.5,High
+Claude 3 Haiku,Long,Bedrock,2.8,236.9,1.0,High
+Claude 3 Haiku,Short,Bedrock,2.2,245.4,0.4,High
+Claude 3 Opus,Long,Bedrock,12.2,242.7,61.1,High
+Claude 3 Opus,Short,Bedrock,8.4,243.2,25.4,High
+Cohere Command R+,Long,Bedrock,7.7,245.7,11.7,High
+Cohere Command R+,Short,Bedrock,7.1,249.9,5.1,High
+Cohere Command Text,Long,Bedrock,12.9,238.7,4.3,High
+Cohere Command Text,Short,Bedrock,9.6,245.6,1.1,High
+Gemini Pro 1.5,Long,Google,5.5,245.7,11.0,High
+Gemini Pro 1.5,Short,Google,5.4,247.5,3.3,High
+Gemini Pro 1,Long,Google,6.0,228.9,1.7,High
+Gemini Pro 1,Short,Google,4.4,247.4,0.6,High
+GPT 3.5 Turbo,Long,OpenAI,4.5,249.9,1.6,High
+GPT 3.5 Turbo,Short,OpenAI,4.2,238.3,0.6,High
+GPT 4 Turbo,Long,OpenAI,12.3,247.6,32.0,High
+GPT 4 Turbo,Short,OpenAI,12.3,250.0,11.7,High
+GPT4-o,Long,OpenAI,5.1,248.4,15.9,High
+GPT4-o,Short,OpenAI,5.0,250.0,5.8,High
+Mistral 7B,Long,Self-host (g5.48xlarge),8.83,242.0,16.5,High
+Mistral 7B,Short,Self-host (g5.48xlarge),8.31,247.0,15.5,High
+LLaMA 3 8B,Long,Self-host (g5.48xlarge),3.76,251.5,7.0,High
+LLaMA 3 8B,Short,Self-host (g5.48xlarge),3.23,243.6,6.0,High
+LLaMA 3 70B,Long,Self-host (p4d.24xlarge),20.1,243.9,67.7,High
+LLaMA 3 70B,Short,Self-host (p4d.24xlarge),29.4,251.2,99.0,High
+Mixtral 8x7B,Long,Self-host (p4d.24xlarge),2.44,248.5,8.22,High
+Mixtral 8x7B,Short,Self-host (p4d.24xlarge),2.41,250.0,8.11,High
+SF-TextBase 7B,Long,Self-host (g5.48xlarge),8.99,248.5,16.80,High
+SF-TextBase 7B,Short,Self-host (g5.48xlarge),8.29,248.7,15.50,High
+SF-TextBase 70B,Long,Self-host (p4de.24xlarge),6.52,253.7,28.17,High
+SF-TextBase 70B,Short,Self-host (p4de.24xlarge),6.24,249.7,26.96,High
+SF-TextSum,Long,Self-host (g5.48xlarge),8.85,244.0,16.55,High
+SF-TextSum,Short,Self-host (g5.48xlarge),8.34,250.4,15.60,High
+XGen 2,Long,Self-host (p4de.24xlarge),3.71,250.0,16.03,High
+XGen 2,Short,Self-host (p4de.24xlarge),2.64,250.0,11.40,High

src/display/utils.py

@@ -25,14 +25,11 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(
     ["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)]
 )
-auto_eval_column_dict.append(
-    ["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True, never_hidden=True)]
-)
-auto_eval_column_dict.append(["use_case_name", ColumnContent, ColumnContent("Use Case Name", "markdown", True)])
+auto_eval_column_dict.append(["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True)])
+auto_eval_column_dict.append(["use_case_name", ColumnContent, ColumnContent("Use Case Name", "markdown", False)])
 auto_eval_column_dict.append(["use_case_type", ColumnContent, ColumnContent("Use Case Type", "markdown", False)])
 
 auto_eval_column_dict.append(["accuracy_method", ColumnContent, ColumnContent("Accuracy Method", "markdown", False)])

src/populate.py

@@ -11,6 +11,8 @@ from src.leaderboard.read_evals import get_raw_eval_results
 def get_leaderboard_df_crm(crm_results_path: str, cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     leaderboard_accuracy_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_accuracy.csv"))
+    sf_finetuned_models = ["SF-TextBase 70B", "SF-TextBase 7B", "SF-TextSum"]
+    leaderboard_accuracy_df = leaderboard_accuracy_df[~leaderboard_accuracy_df["Model Name"].isin(sf_finetuned_models)]
     # leaderboard_accuracy_df = leaderboard_accuracy_df.sort_values(
     #     by=[AutoEvalColumn.accuracy_metric_average.name], ascending=False
     # )