import gradio as gr import pandas as pd from utils_display import make_clickable_model banner_url = "https://huggingface.co/spaces/deepdml/open_universal_arabic_quranic_asr_leaderboard/main/banner.png" BANNER = f'
Banner
' INTRODUCTION_TEXT = "πŸ“–**Open Universal Arabic Quranic ASR Leaderboard**πŸ“– benchmarks multi-dialect Arabic Quranic ASR models on various multi-dialect datasets.
Apart from the WER%/CER% for each test set, we also report the Average WER%/CER% and rank the models based on the Average WER, from lowest to highest.
To reproduce the benchmark numbers and request a model that is not listed, you can launch an issue/PR in our [GitHub repo](https://github.com/Natural-Language-Processing-Elm/open_universal_arabic_asr_leaderboard)😊.
For more detailed analysis such as models' robustness, speaker adaption, model efficiency and memory usage, please check our [paper](https://arxiv.org/pdf/2412.13788)." CITATION_BUTTON_TEXT = """ @article{wang2024open, title={Open Universal Arabic Quranic ASR Leaderboard}, author={Jimenez, David}, year={2025} } """ METRICS_TAB_TEXT = """ ## Metrics We report both the Word Error Rate (WER) and Character Error Rate (CER). ## Reproduction The Open Universal Arabic Quranic ASR Leaderboard will be a continuous benchmark project. \nWe open-source the evaluation scripts at our [GitHub repo](https://github.com/Natural-Language-Processing-Elm/open_universal_arabic_asr_leaderboard). \nPlease launch a discussion in our GitHub repo to let us know if you want to learn about the performance of a new model. ## Benchmark datasets | Test Set | Num Samples | |-------------------------------------------------------------------------------------------------|-------------| | [Tarteel AI's EveryAyah](https://huggingface.co/datasets/tarteel-ai/everyayah) | 23.473 | ## In-depth Analysis We also provide a comprehensive analysis of the models' robustness, speaker adaptation, inference efficiency and memory consumption. \nPlease check our [paper](https://arxiv.org/pdf/2412.13788) to learn more. """ def styled_message(message): return f"

{message}

" UPDATES = "Oct 20th 2025:[Created repo]
" results = { "Model": ["tarteel-ai/whisper-tiny-ar-quran", "Habib-HF/tarbiyah-ai-whisper-medium-merged", "nvidia/stt_ar_fastconformer_hybrid_large_pcd_v1.0", "IbrahimSalah/Wav2vecLarge_quran_syllables_recognition", "facebook/mms-1b-all", "facebook/hf-seamless-m4t-medium", "facebook/seamless-m4t-v2-large", "IJyad/whisper-large-v3-Tarteel"], "Average WER⬇️": [4.99, 81.59, 1.33, 281.27, 48.13, 25.77, 26.2, 25.39], "Average CER": [1.82, 76.65, 0.49, 73.18, 16.58, 15.38, 18.08, 17.75], "Tarteel WER": [4.99, 81.59, 1.33, 281.27, 48.13, 25.77, 26.2, 25.39], "Tarteel CER": [1.82, 76.65, 0.49, 73.18, 16.58, 15.38, 18.08, 17.75], } original_df = pd.DataFrame(results) # original_df["Model"] = original_df["Model"].apply(lambda x: x.replace(x, make_clickable_model(x))) original_df.sort_values(by="Average WER⬇️", inplace=True) TYPES = ['str', 'number', 'number', 'number', 'number'] LEADERBOARD_CSS = """ html, body { overflow-y: auto !important; } #leaderboard-table th .header-content { min-width: 150px; white-space: nowrap; } """ def request_model(model_text): return styled_message("πŸ€— Please launch a discussion in our GitHub repo, thank you. πŸ€—") with gr.Blocks(fill_width=False, fill_height=False, css=LEADERBOARD_CSS) as demo: gr.HTML(BANNER, elem_id="banner") gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("πŸ… Leaderboard", elem_id="od-benchmark-tab-table", id=0): leaderboard_table = gr.Dataframe( value=original_df, datatype=TYPES, elem_id="leaderboard-table", interactive=False, visible=True, ) with gr.TabItem("πŸ“ˆ Metrics", elem_id="od-benchmark-tab-table", id=1): gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text") with gr.TabItem("βœ‰οΈβœ¨ Request a model here!", elem_id="od-benchmark-tab-table", id=2): with gr.Column(): gr.Markdown("# βœ‰οΈβœ¨ Request results for a new model here!", elem_classes="markdown-text") model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)") mdw_submission_result = gr.Markdown() btn_submit = gr.Button(value="πŸš€ Request") btn_submit.click(request_model, [model_name_textbox], mdw_submission_result) gr.Markdown(UPDATES, elem_classes="markdown-text") with gr.Row(): with gr.Accordion("πŸ“™ Citation", open=False): gr.Textbox( value=CITATION_BUTTON_TEXT, lines=7, label="Copy the BibTeX snippet to cite this source", elem_id="citation-button", show_copy_button=True, ) demo.launch(allowed_paths=["banner.png"], ssr_mode=False)