Commit
Β·
7939a4f
1
Parent(s):
1ddac98
updated naming
Browse files- README.md +7 -7
- app.py +3 -3
- demo.py +2 -2
- results/arima_submission/metadata.json +1 -1
- results/exponential_smoothing_submission/metadata.json +1 -1
- results/historical_inertia_submission/metadata.json +1 -1
- results/linear_regression_submission/metadata.json +2 -2
- results/linear_trend_submission/metadata.json +1 -1
- results/mean_submission/metadata.json +1 -1
- src/__init__.py +1 -1
- src/about.py +6 -6
- src/display/css_html_js.py +2 -2
- src/display/utils.py +1 -1
- src/envs.py +4 -4
- src/load_results.py +1 -1
- src/populate.py +1 -1
- src/utils.py +1 -1
README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
emoji: π
|
| 4 |
colorFrom: pink
|
| 5 |
colorTo: indigo
|
|
@@ -7,17 +7,17 @@ sdk: gradio
|
|
| 7 |
sdk_version: 5.49.0
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
-
short_description: Leaderboard for
|
| 11 |
license: mit
|
| 12 |
---
|
| 13 |
|
| 14 |
-
# π
|
| 15 |
|
| 16 |
-
Welcome to the
|
| 17 |
|
| 18 |
-
## π About
|
| 19 |
|
| 20 |
-
|
| 21 |
|
| 22 |
## π― Evaluation Metrics
|
| 23 |
|
|
@@ -52,7 +52,7 @@ The leaderboard uses the following metrics to evaluate model performance:
|
|
| 52 |
5. Click "π Submit Model" to add your results
|
| 53 |
|
| 54 |
### Dataset Information
|
| 55 |
-
- Visit the "π Dataset Info" tab for detailed information about
|
| 56 |
- Check submission guidelines and evaluation protocols
|
| 57 |
|
| 58 |
### Statistics
|
|
|
|
| 1 |
---
|
| 2 |
+
title: MUSEval Leaderboard
|
| 3 |
emoji: π
|
| 4 |
colorFrom: pink
|
| 5 |
colorTo: indigo
|
|
|
|
| 7 |
sdk_version: 5.49.0
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
+
short_description: Leaderboard for MUSEval Dataset
|
| 11 |
license: mit
|
| 12 |
---
|
| 13 |
|
| 14 |
+
# π MUSEval Leaderboard
|
| 15 |
|
| 16 |
+
Welcome to the MUSEval (Multivariate Time Series Dataset) Leaderboard! This leaderboard tracks and compares the performance of different models on multivariate time series forecasting tasks.
|
| 17 |
|
| 18 |
+
## π About MUSEval
|
| 19 |
|
| 20 |
+
MUSEval is a comprehensive multivariate time series dataset designed for forecasting tasks. The dataset contains multiple time series with various characteristics and complexities, making it an ideal benchmark for evaluating time series forecasting models.
|
| 21 |
|
| 22 |
## π― Evaluation Metrics
|
| 23 |
|
|
|
|
| 52 |
5. Click "π Submit Model" to add your results
|
| 53 |
|
| 54 |
### Dataset Information
|
| 55 |
+
- Visit the "π Dataset Info" tab for detailed information about MUSEval
|
| 56 |
- Check submission guidelines and evaluation protocols
|
| 57 |
|
| 58 |
### Statistics
|
app.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
Synthefy
|
| 3 |
Following GIFT-Eval import structure with custom layout
|
| 4 |
"""
|
| 5 |
|
|
@@ -188,7 +188,7 @@ def create_leaderboard_interface():
|
|
| 188 |
)
|
| 189 |
|
| 190 |
# About section
|
| 191 |
-
with gr.Accordion("π About
|
| 192 |
gr.Markdown(BENCHMARKS_TEXT, elem_classes="markdown-text", elem_id="about-text")
|
| 193 |
|
| 194 |
# Citation section
|
|
@@ -208,7 +208,7 @@ def create_leaderboard_interface():
|
|
| 208 |
gr.HTML("""
|
| 209 |
<div style="text-align: center; padding: 20px;">
|
| 210 |
<h3>Submit by creating a pull request with your model's performance here:</h3>
|
| 211 |
-
<a href='https://github.com/Synthefy/
|
| 212 |
target='_blank'
|
| 213 |
style='display: inline-block;
|
| 214 |
background-color: #FF6B6B;
|
|
|
|
| 1 |
"""
|
| 2 |
+
Synthefy MUSEval Leaderboard - Main Gradio Application
|
| 3 |
Following GIFT-Eval import structure with custom layout
|
| 4 |
"""
|
| 5 |
|
|
|
|
| 188 |
)
|
| 189 |
|
| 190 |
# About section
|
| 191 |
+
with gr.Accordion("π About MUSEval Leaderboard", open=False, elem_id="about-accordion"):
|
| 192 |
gr.Markdown(BENCHMARKS_TEXT, elem_classes="markdown-text", elem_id="about-text")
|
| 193 |
|
| 194 |
# Citation section
|
|
|
|
| 208 |
gr.HTML("""
|
| 209 |
<div style="text-align: center; padding: 20px;">
|
| 210 |
<h3>Submit by creating a pull request with your model's performance here:</h3>
|
| 211 |
+
<a href='https://github.com/Synthefy/MUSEval'
|
| 212 |
target='_blank'
|
| 213 |
style='display: inline-block;
|
| 214 |
background-color: #FF6B6B;
|
demo.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
|
| 3 |
Imports from app.py to ensure identical functionality, loads a local demo leaderboard
|
| 4 |
"""
|
| 5 |
|
|
@@ -29,7 +29,7 @@ demo = create_leaderboard_interface()
|
|
| 29 |
|
| 30 |
# Launch the demo
|
| 31 |
if __name__ == "__main__":
|
| 32 |
-
print("π¨
|
| 33 |
print("=" * 50)
|
| 34 |
|
| 35 |
try:
|
|
|
|
| 1 |
"""
|
| 2 |
+
MUSEval Leaderboard - Local Demo
|
| 3 |
Imports from app.py to ensure identical functionality, loads a local demo leaderboard
|
| 4 |
"""
|
| 5 |
|
|
|
|
| 29 |
|
| 30 |
# Launch the demo
|
| 31 |
if __name__ == "__main__":
|
| 32 |
+
print("π¨ MUSEval Leaderboard Local Demo")
|
| 33 |
print("=" * 50)
|
| 34 |
|
| 35 |
try:
|
results/arima_submission/metadata.json
CHANGED
|
@@ -5,6 +5,6 @@
|
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
"paper_url": "https://www.wiley.com/en-us/Time+Series+Analysis%3A+Forecasting+and+Control%2C+5th+Edition-p-9781118675021",
|
| 8 |
-
"code_url": "https://github.com/Synthefy/
|
| 9 |
"description": "AutoRegressive Integrated Moving Average (ARIMA) model for time series forecasting. Uses autoregression, differencing, and moving averages to capture trends and patterns in multivariate time series data."
|
| 10 |
}
|
|
|
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
"paper_url": "https://www.wiley.com/en-us/Time+Series+Analysis%3A+Forecasting+and+Control%2C+5th+Edition-p-9781118675021",
|
| 8 |
+
"code_url": "https://github.com/Synthefy/MUSEval/blob/main/src/museval/baselines/arima_forecast.py",
|
| 9 |
"description": "AutoRegressive Integrated Moving Average (ARIMA) model for time series forecasting. Uses autoregression, differencing, and moving averages to capture trends and patterns in multivariate time series data."
|
| 10 |
}
|
results/exponential_smoothing_submission/metadata.json
CHANGED
|
@@ -5,6 +5,6 @@
|
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
"paper_url": "https://www.bauer.uh.edu/gardner/Exp-Sm-1985.pdf",
|
| 8 |
-
"code_url": "https://github.com/Synthefy/
|
| 9 |
"description": "Exponential Smoothing method for time series forecasting. Applies exponentially decreasing weights to historical observations."
|
| 10 |
}
|
|
|
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
"paper_url": "https://www.bauer.uh.edu/gardner/Exp-Sm-1985.pdf",
|
| 8 |
+
"code_url": "https://github.com/Synthefy/MUSEval/blob/main/src/museval/baselines/exponential_smoothing.py",
|
| 9 |
"description": "Exponential Smoothing method for time series forecasting. Applies exponentially decreasing weights to historical observations."
|
| 10 |
}
|
results/historical_inertia_submission/metadata.json
CHANGED
|
@@ -5,6 +5,6 @@
|
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
"paper_url": "https://arxiv.org/pdf/2103.16349",
|
| 8 |
-
"code_url": "https://github.com/Synthefy/
|
| 9 |
"description": "Historical Inertia baseline that repeates the recent history as the forecast."
|
| 10 |
}
|
|
|
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
"paper_url": "https://arxiv.org/pdf/2103.16349",
|
| 8 |
+
"code_url": "https://github.com/Synthefy/MUSEval/blob/main/src/museval/baselines/historical_inertia.py",
|
| 9 |
"description": "Historical Inertia baseline that repeates the recent history as the forecast."
|
| 10 |
}
|
results/linear_regression_submission/metadata.json
CHANGED
|
@@ -4,7 +4,7 @@
|
|
| 4 |
"submission_date": "2025-10-10",
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
-
"paper_url": "https://github.com/Synthefy/
|
| 8 |
-
"code_url": "https://github.com/Synthefy/
|
| 9 |
"description": "Linear Regression model over the history used to extrapolate the future, includes correlates for multivariate."
|
| 10 |
}
|
|
|
|
| 4 |
"submission_date": "2025-10-10",
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
+
"paper_url": "https://github.com/Synthefy/MUSEval",
|
| 8 |
+
"code_url": "https://github.com/Synthefy/MUSEval/blob/main/src/museval/baselines/linear_regression.py",
|
| 9 |
"description": "Linear Regression model over the history used to extrapolate the future, includes correlates for multivariate."
|
| 10 |
}
|
results/linear_trend_submission/metadata.json
CHANGED
|
@@ -4,6 +4,6 @@
|
|
| 4 |
"submission_date": "2025-10-10",
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
-
"code_url": "https://github.com/Synthefy/
|
| 8 |
"description": "Linear Trend model for time series forecasting."
|
| 9 |
}
|
|
|
|
| 4 |
"submission_date": "2025-10-10",
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
"dataset_version": "v1.0",
|
| 7 |
+
"code_url": "https://github.com/Synthefy/MUSEval/blob/main/src/museval/baselines/linear_trend.py",
|
| 8 |
"description": "Linear Trend model for time series forecasting."
|
| 9 |
}
|
results/mean_submission/metadata.json
CHANGED
|
@@ -3,6 +3,6 @@
|
|
| 3 |
"submitter": "Synthefy",
|
| 4 |
"submission_date": "2025-10-10",
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
-
"code_url": "https://github.com/Synthefy/
|
| 7 |
"description": "Uses the mean of the historical values as the prediction for all future values."
|
| 8 |
}
|
|
|
|
| 3 |
"submitter": "Synthefy",
|
| 4 |
"submission_date": "2025-10-10",
|
| 5 |
"task": "multivariate_forecasting",
|
| 6 |
+
"code_url": "https://github.com/Synthefy/MUSEval/blob/main/src/museval/baselines/mean_forecast.py",
|
| 7 |
"description": "Uses the mean of the historical values as the prediction for all future values."
|
| 8 |
}
|
src/__init__.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
|
| 3 |
"""
|
| 4 |
|
| 5 |
from .load_results import (
|
|
|
|
| 1 |
"""
|
| 2 |
+
MUSEval Leaderboard source package
|
| 3 |
"""
|
| 4 |
|
| 5 |
from .load_results import (
|
src/about.py
CHANGED
|
@@ -1,18 +1,18 @@
|
|
| 1 |
"""
|
| 2 |
-
Text constants for
|
| 3 |
"""
|
| 4 |
|
| 5 |
TITLE = """
|
| 6 |
<div id="main-title" style="text-align: center;">
|
| 7 |
-
<h1 style="font-size: 30px; margin-bottom: 15px; font-weight: bold;">π
|
| 8 |
</div>
|
| 9 |
"""
|
| 10 |
|
| 11 |
INTRODUCTION_TEXT = """
|
| 12 |
<div style="font-size: 16px; line-height: 1.6;">
|
| 13 |
-
Welcome to the
|
| 14 |
Use the filters below to explore results by different criteria and compare model performance across various domains and categories. For additional details on the models, click on the models to access the Model Inspector below the table.
|
| 15 |
-
Metrics are explained in "About
|
| 16 |
This leaderboard determines the best performing model for multivariate time series forecasting tasks, as measured by the lowest Mean Absolute Percentage Error (MAPE).
|
| 17 |
High performance on these datasets provides evidence that a model can utilize historical time series relationships to make accurate predictions.
|
| 18 |
|
|
@@ -46,8 +46,8 @@ BENCHMARKS_TEXT = """
|
|
| 46 |
## Contact & Support
|
| 47 |
|
| 48 |
For questions about the dataset or leaderboard:
|
| 49 |
-
- **Issues**: Report issues on the [GitHub repository](https://github.com/Synthefy/
|
| 50 |
-
- **Dataset**: Try the dataset yourself on [Hugging Face](https://huggingface.co/datasets/Synthefy/
|
| 51 |
|
| 52 |
## Leaderboard Information
|
| 53 |
|
|
|
|
| 1 |
"""
|
| 2 |
+
Text constants for MUSEval Leaderboard
|
| 3 |
"""
|
| 4 |
|
| 5 |
TITLE = """
|
| 6 |
<div id="main-title" style="text-align: center;">
|
| 7 |
+
<h1 style="font-size: 30px; margin-bottom: 15px; font-weight: bold;">π MUSEval Leaderboard</h1>
|
| 8 |
</div>
|
| 9 |
"""
|
| 10 |
|
| 11 |
INTRODUCTION_TEXT = """
|
| 12 |
<div style="font-size: 16px; line-height: 1.6;">
|
| 13 |
+
Welcome to the MUSEval Leaderboard! This leaderboard provides comprehensive evaluation results of multivariate time series forecasting. Rows are models and columns are performance metrics.
|
| 14 |
Use the filters below to explore results by different criteria and compare model performance across various domains and categories. For additional details on the models, click on the models to access the Model Inspector below the table.
|
| 15 |
+
Metrics are explained in "About MUSEval Leaderboard" below the table. Submissions can be added at <a href="https://github.com/Synthefy/MUSEval">this github repository</a>.
|
| 16 |
This leaderboard determines the best performing model for multivariate time series forecasting tasks, as measured by the lowest Mean Absolute Percentage Error (MAPE).
|
| 17 |
High performance on these datasets provides evidence that a model can utilize historical time series relationships to make accurate predictions.
|
| 18 |
|
|
|
|
| 46 |
## Contact & Support
|
| 47 |
|
| 48 |
For questions about the dataset or leaderboard:
|
| 49 |
+
- **Issues**: Report issues on the [GitHub repository](https://github.com/Synthefy/MUSEval)
|
| 50 |
+
- **Dataset**: Try the dataset yourself on [Hugging Face](https://huggingface.co/datasets/Synthefy/MUSEval)
|
| 51 |
|
| 52 |
## Leaderboard Information
|
| 53 |
|
src/display/css_html_js.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
"""
|
| 2 |
-
CSS and styling for
|
| 3 |
"""
|
| 4 |
|
| 5 |
custom_css = """
|
| 6 |
-
/* Custom styling for
|
| 7 |
|
| 8 |
/* Main title styling */
|
| 9 |
#main-title h1 {
|
|
|
|
| 1 |
"""
|
| 2 |
+
CSS and styling for MUSEval Leaderboard
|
| 3 |
"""
|
| 4 |
|
| 5 |
custom_css = """
|
| 6 |
+
/* Custom styling for MUSEval Leaderboard */
|
| 7 |
|
| 8 |
/* Main title styling */
|
| 9 |
#main-title h1 {
|
src/display/utils.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
Display utilities and column definitions for
|
| 3 |
"""
|
| 4 |
|
| 5 |
from dataclasses import dataclass
|
|
|
|
| 1 |
"""
|
| 2 |
+
Display utilities and column definitions for MUSEval Leaderboard
|
| 3 |
"""
|
| 4 |
|
| 5 |
from dataclasses import dataclass
|
src/envs.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
Environment configuration for
|
| 3 |
"""
|
| 4 |
|
| 5 |
import os
|
|
@@ -12,9 +12,9 @@ class API:
|
|
| 12 |
print(f"Restarting space: {repo_id}")
|
| 13 |
|
| 14 |
# Repository configuration
|
| 15 |
-
REPO_ID = "
|
| 16 |
-
QUEUE_REPO = "
|
| 17 |
-
RESULTS_REPO = "
|
| 18 |
|
| 19 |
# Paths
|
| 20 |
EVAL_REQUESTS_PATH = "eval_requests"
|
|
|
|
| 1 |
"""
|
| 2 |
+
Environment configuration for MUSEval Leaderboard
|
| 3 |
"""
|
| 4 |
|
| 5 |
import os
|
|
|
|
| 12 |
print(f"Restarting space: {repo_id}")
|
| 13 |
|
| 14 |
# Repository configuration
|
| 15 |
+
REPO_ID = "museval-leaderboard"
|
| 16 |
+
QUEUE_REPO = "museval-queue"
|
| 17 |
+
RESULTS_REPO = "museval-results"
|
| 18 |
|
| 19 |
# Paths
|
| 20 |
EVAL_REQUESTS_PATH = "eval_requests"
|
src/load_results.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
Data loading utilities for
|
| 3 |
"""
|
| 4 |
|
| 5 |
import json
|
|
|
|
| 1 |
"""
|
| 2 |
+
Data loading utilities for MUSEval Leaderboard
|
| 3 |
"""
|
| 4 |
|
| 5 |
import json
|
src/populate.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
Data population functions for
|
| 3 |
"""
|
| 4 |
|
| 5 |
import pandas as pd
|
|
|
|
| 1 |
"""
|
| 2 |
+
Data population functions for MUSEval Leaderboard
|
| 3 |
"""
|
| 4 |
|
| 5 |
import pandas as pd
|
src/utils.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
Utility functions for
|
| 3 |
"""
|
| 4 |
|
| 5 |
import pandas as pd
|
|
|
|
| 1 |
"""
|
| 2 |
+
Utility functions for MUSEval Leaderboard
|
| 3 |
"""
|
| 4 |
|
| 5 |
import pandas as pd
|