Create app.py

app.py

@@ -0,0 +1,146 @@
+import gradio as gr
+import random
+import numpy as np
+import pandas as pd
+from datasets import load_dataset
+from sentence_transformers import CrossEncoder
+from sklearn.metrics import average_precision_score
+import matplotlib.pyplot as plt
+import torch
+import spaces
+
+# Check for GPU support and configure appropriately
+device = "cuda" if torch.cuda.is_available() else "cpu"
+zero = torch.Tensor([0]).to(device)
+print(f"Device being used: {zero.device}")
+
+# Define evaluation metrics
+def mean_reciprocal_rank(relevance_labels, scores):
+    sorted_indices = np.argsort(scores)[::-1]
+    for rank, idx in enumerate(sorted_indices, start=1):
+        if relevance_labels[idx] == 1:
+            return 1 / rank
+    return 0
+
+def mean_average_precision(relevance_labels, scores):
+    return average_precision_score(relevance_labels, scores)
+
+def ndcg_at_k(relevance_labels, scores, k=10):
+    sorted_indices = np.argsort(scores)[::-1]
+    relevance_sorted = np.take(relevance_labels, sorted_indices[:k])
+    dcg = sum(rel / np.log2(rank + 2) for rank, rel in enumerate(relevance_sorted))
+    idcg = sum(1 / np.log2(rank + 2) for rank in range(min(k, sum(relevance_labels))))
+    return dcg / idcg if idcg > 0 else 0
+
+# Load datasets
+datasets = {
+    "Relevance_Labels_Dataset": load_dataset("NAMAA-Space/Ar-Reranking-Eval")["train"],
+    "Positive_Negatives_Dataset": load_dataset("NAMAA-Space/Arabic-Reranking-Triplet-5-Eval")["train"]
+}
+
+@spaces.GPU
+def evaluate_model_with_insights(model_name):
+    model = CrossEncoder(model_name, device=device)
+    results = []
+    sample_outputs = []
+
+    for dataset_name, dataset in datasets.items():
+        all_mrr, all_map, all_ndcg = [], [], []
+        dataset_samples = []
+
+        if 'candidate_document' in dataset.column_names:
+            grouped_data = dataset.to_pandas().groupby("query")
+            for query, group in grouped_data:
+                candidate_texts = group['candidate_document'].tolist()
+                relevance_labels = group['relevance_label'].tolist()
+                pairs = [(query, doc) for doc in candidate_texts]
+                scores = model.predict(pairs)
+
+                # Collecting top-5 results for display
+                sorted_indices = np.argsort(scores)[::-1]
+                top_docs = [(candidate_texts[i], scores[i], relevance_labels[i]) for i in sorted_indices[:5]]
+                dataset_samples.append({
+                    "Query": query,
+                    "Top 5 Candidates": top_docs
+                })
+
+                # Metrics
+                all_mrr.append(mean_reciprocal_rank(relevance_labels, scores))
+                all_map.append(mean_average_precision(relevance_labels, scores))
+                all_ndcg.append(ndcg_at_k(relevance_labels, scores, k=10))
+        else:
+            for entry in dataset:
+                query = entry['query']
+                candidate_texts = [entry['positive'], entry['negative1'], entry['negative2'], entry['negative3'], entry['negative4']]
+                relevance_labels = [1, 0, 0, 0, 0]
+                pairs = [(query, doc) for doc in candidate_texts]
+                scores = model.predict(pairs)
+
+                # Collecting top-5 results for display
+                sorted_indices = np.argsort(scores)[::-1]
+                top_docs = [(candidate_texts[i], scores[i], relevance_labels[i]) for i in sorted_indices[:5]]
+                dataset_samples.append({
+                    "Query": query,
+                    "Top 5 Candidates": top_docs
+                })
+
+                # Metrics
+                all_mrr.append(mean_reciprocal_rank(relevance_labels, scores))
+                all_map.append(mean_average_precision(relevance_labels, scores))
+                all_ndcg.append(ndcg_at_k(relevance_labels, scores, k=10))
+        
+        # Metrics for this dataset
+        results.append({
+            "Dataset": dataset_name,
+            "MRR": np.mean(all_mrr),
+            "MAP": np.mean(all_map),
+            "nDCG@10": np.mean(all_ndcg)
+        })
+
+        # Collect sample outputs for inspection
+        sample_outputs.extend(dataset_samples)
+
+    results_df = pd.DataFrame(results)
+
+    # Plot results as a bar chart
+    fig, ax = plt.subplots(figsize=(8, 6))
+    results_df.plot(kind='bar', x='Dataset', y=['MRR', 'MAP', 'nDCG@10'], ax=ax)
+    ax.set_title(f"Evaluation Results for {model_name}")
+    ax.set_ylabel("Score")
+    plt.xticks(rotation=0)
+
+    return results_df, fig, sample_outputs
+
+# Gradio app interface
+def gradio_app_with_insights(model_name):
+    results_df, chart, samples = evaluate_model_with_insights(model_name)
+    sample_display = []
+    for sample in samples:
+        sample_display.append(f"Query: {sample['Query']}")
+        for doc, score, label in sample["Top 5 Candidates"]:
+            sample_display.append(f"    Doc: {doc[:50]}... | Score: {score:.2f} | Relevance: {label}")
+        sample_display.append("\n")
+    return results_df, chart, "\n".join(sample_display)
+
+interface = gr.Interface(
+    fn=gradio_app_with_insights,
+    inputs=gr.Textbox(label="Enter Model Name", placeholder="e.g., NAMAA-Space/GATE-Reranker-V1"),
+    outputs=[
+        gr.Dataframe(label="Evaluation Results"),
+        gr.Plot(label="Evaluation Metrics Chart"),
+        gr.Textbox(label="Sample Reranking Insights", lines=15)
+    ],
+    title="Arabic Reranking Model Evaluation and Insights",
+    description=(
+        "This app evaluates Arabic reranking models on two datasets:\n"
+        "1. **Relevance Labels Dataset**\n"
+        "2. **Positive-Negatives Dataset**\n\n"
+        "### Metrics Used:\n"
+        "- **MRR (Mean Reciprocal Rank)**: Measures how quickly the first relevant document appears.\n"
+        "- **MAP (Mean Average Precision)**: Reflects ranking quality across all relevant documents.\n"
+        "- **nDCG@10 (Normalized Discounted Cumulative Gain)**: Focuses on the ranking of relevant documents in the top-10.\n\n"
+        "Input a model name to evaluate its performance, view metrics, and examine sample reranking results."
+    )
+)
+
+interface.launch(debug=True)