Spaces:
Runtime error
Runtime error
| import start | |
| import gradio as gr | |
| import pandas as pd | |
| from glob import glob | |
| from pathlib import Path | |
| from tabs.dashboard import df | |
| from tabs.faq import ( | |
| about_olas_predict_benchmark, | |
| about_olas_predict, | |
| about_the_dataset, | |
| about_the_tools, | |
| ) | |
| from tabs.howto_benchmark import how_to_run | |
| # disabling temporarily | |
| # from tabs.run_benchmark import run_benchmark_main | |
| demo = gr.Blocks() | |
| def run_benchmark_gradio( | |
| tool_name, | |
| model_name, | |
| num_questions, | |
| openai_api_key, | |
| anthropic_api_key, | |
| openrouter_api_key, | |
| ): | |
| """Run the benchmark using inputs.""" | |
| if tool_name is None: | |
| return "Please enter the name of your tool." | |
| if ( | |
| openai_api_key is None | |
| and anthropic_api_key is None | |
| and openrouter_api_key is None | |
| ): | |
| return "Please enter either OpenAI or Anthropic or OpenRouter API key." | |
| result = run_benchmark_main( | |
| tool_name, | |
| model_name, | |
| num_questions, | |
| openai_api_key, | |
| anthropic_api_key, | |
| openrouter_api_key, | |
| ) | |
| if result == "completed": | |
| # get the results file in the results directory | |
| fns = glob("results/*.csv") | |
| print(f"Number of files in results directory: {len(fns)}") | |
| # convert to Path | |
| files = [Path(file) for file in fns] | |
| # get results and summary files | |
| results_files = [file for file in files if "results" in file.name] | |
| # the other file is the summary file | |
| summary_files = [file for file in files if "summary" in file.name] | |
| print(results_files, summary_files) | |
| # get the path with results | |
| results_df = pd.read_csv(results_files[0]) | |
| summary_df = pd.read_csv(summary_files[0]) | |
| # make sure all df float values are rounded to 4 decimal places | |
| results_df = results_df.round(4) | |
| summary_df = summary_df.round(4) | |
| return gr.Dataframe(value=results_df), gr.Dataframe(value=summary_df) | |
| return gr.Textbox( | |
| label="Benchmark Result", value=result, interactive=False | |
| ), gr.Textbox(label="Summary", value="") | |
| with demo: | |
| gr.HTML("<h1>Olas Predict Benchmark</hjson>") | |
| gr.Markdown( | |
| "Leaderboard showing the performance of Olas Predict tools on the Autocast dataset and overview of the project." | |
| ) | |
| with gr.Tabs() as tabs: | |
| # first tab - leaderboard | |
| with gr.TabItem("🏅 Benchmark Leaderboard", id=0): | |
| gr.components.Dataframe( | |
| value=df, | |
| ) | |
| # second tab - about | |
| with gr.TabItem("ℹ️ About"): | |
| with gr.Row(): | |
| with gr.Accordion("About the Benchmark", open=False): | |
| gr.Markdown(about_olas_predict_benchmark) | |
| with gr.Row(): | |
| with gr.Accordion("About the Tools", open=False): | |
| gr.Markdown(about_the_tools) | |
| with gr.Row(): | |
| with gr.Accordion("About the Autocast Dataset", open=False): | |
| gr.Markdown(about_the_dataset) | |
| with gr.Row(): | |
| with gr.Accordion("About Olas", open=False): | |
| gr.Markdown(about_olas_predict) | |
| # third tab - how to run the benchmark | |
| with gr.TabItem("🚀 Contribute"): | |
| gr.Markdown(how_to_run) | |
| # fourth tab - run the benchmark | |
| # with gr.TabItem("🔥 Run the Benchmark"): | |
| # with gr.Row(): | |
| # tool_name = gr.Dropdown( | |
| # [ | |
| # "prediction-offline", | |
| # "prediction-online", | |
| # # "prediction-online-summarized-info", | |
| # # "prediction-offline-sme", | |
| # # "prediction-online-sme", | |
| # "prediction-request-rag", | |
| # "prediction-request-reasoning", | |
| # # "prediction-url-cot-claude", | |
| # # "prediction-request-rag-cohere", | |
| # # "prediction-with-research-conservative", | |
| # # "prediction-with-research-bold", | |
| # ], | |
| # label="Tool Name", | |
| # info="Choose the tool to run", | |
| # ) | |
| # model_name = gr.Dropdown( | |
| # [ | |
| # "gpt-3.5-turbo-0125", | |
| # "gpt-4-0125-preview", | |
| # "claude-3-haiku-20240307", | |
| # "claude-3-sonnet-20240229", | |
| # "claude-3-opus-20240229", | |
| # "databricks/dbrx-instruct:nitro", | |
| # "nousresearch/nous-hermes-2-mixtral-8x7b-sft", | |
| # # "cohere/command-r-plus", | |
| # ], | |
| # label="Model Name", | |
| # info="Choose the model to use", | |
| # ) | |
| # with gr.Row(): | |
| # openai_api_key = gr.Textbox( | |
| # label="OpenAI API Key", | |
| # placeholder="Enter your OpenAI API key here", | |
| # type="password", | |
| # ) | |
| # anthropic_api_key = gr.Textbox( | |
| # label="Anthropic API Key", | |
| # placeholder="Enter your Anthropic API key here", | |
| # type="password", | |
| # ) | |
| # openrouter_api_key = gr.Textbox( | |
| # label="OpenRouter API Key", | |
| # placeholder="Enter your OpenRouter API key here", | |
| # type="password", | |
| # ) | |
| # with gr.Row(): | |
| # num_questions = gr.Slider( | |
| # minimum=1, | |
| # maximum=340, | |
| # value=10, | |
| # label="Number of questions to run the benchmark on", | |
| # ) | |
| # with gr.Row(): | |
| # run_button = gr.Button("Run Benchmark") | |
| # with gr.Row(): | |
| # with gr.Accordion("Results", open=True): | |
| # result = gr.Dataframe() | |
| # with gr.Row(): | |
| # with gr.Accordion("Summary", open=False): | |
| # summary = gr.Dataframe() | |
| # run_button.click( | |
| # run_benchmark_gradio, | |
| # inputs=[ | |
| # tool_name, | |
| # model_name, | |
| # num_questions, | |
| # openai_api_key, | |
| # anthropic_api_key, | |
| # openrouter_api_key, | |
| # ], | |
| # outputs=[result, summary], | |
| # ) | |
| demo.queue(default_concurrency_limit=40).launch() | |