order and duplicate fix

intermediate_leaderboard.py

@@ -13,7 +13,7 @@ import pandas as pd
 def validate_hf_username(username):
     username = str(username).strip()
     hf_url = f"https://huggingface.co/{username}"
-    return check_page_exists(hf_url, delay=1)
+    return check_page_exists(hf_url, delay=1, max_retries=10)
     # return True  # For testing purposes, assume all usernames are valid
 
 def validate_model_details(tag):
@@ -37,6 +37,8 @@ def make_intermediate_lb():
         test_repo=results_repo_test
     )
 
+    # Make all usernames lowercase
+    df_latest_raw["hf_username"] = df_latest_raw["hf_username"].str.lower()
 
     # HF username validation
     hf_usernames = df_latest_raw["hf_username"].unique()
@@ -53,11 +55,12 @@ def make_intermediate_lb():
     # make sure to only keep the latest submission per user for the 'Average' endpoint
     df_latest_raw["submission_time"] = pd.to_datetime(df_latest_raw["submission_time"])
     df_latest_raw = df_latest_raw.query("Endpoint == 'Average'")
-    df_latest_raw['latest_time_per_user'] = df_latest_raw.groupby('user')['submission_time'].transform('max')
+    df_latest_raw['latest_time_per_user'] = df_latest_raw.groupby('hf_username')['submission_time'].transform('max')
     latest_submissions_df = df_latest_raw[df_latest_raw['submission_time'] == df_latest_raw['latest_time_per_user']].copy()
-
+    # Fix to order by the mean RAE and not the RAE of all samples (slight missmatch for some users)
+    latest_submissions_df['mean_RAE'] = latest_submissions_df.groupby('hf_username')['RAE'].transform('mean')    
     latest_submissions_df = latest_submissions_df.sort_values(
-        ['RAE'], ascending=True
+        by=['mean_RAE', 'Sample'], ascending=True
     ).reset_index(drop=True)
 
     # Get the unique users in the order of their first appearance
@@ -75,9 +78,6 @@ def make_intermediate_lb():
     # Apply the mapping to create a new column with prefixed usernames
     latest_submissions_df['user'] = latest_submissions_df['user'].map(user_mapping)
 
-
-
-
     # Perform Tukey's HSD test
     tukey = pairwise_tukeyhsd(endog=latest_submissions_df['RAE'], groups=latest_submissions_df['user'], alpha=0.05)
     tukey_df = pd.DataFrame(data=tukey._results_table.data[1:], 
@@ -89,7 +89,6 @@ def make_intermediate_lb():
     cld_df = pd.DataFrame(cld_dict.items(),columns=["group","letter"]).sort_values("group")
     cld_df.letter = [",".join(x) for x in cld_df.letter]
     cld_df["user"] = cld_df.group
-    cld_df["user_fixed"] = cld_df.group.str.split("___").str[1]
 
     # clean up CLD letters for extended alphabet (i.e with @ symbols)
     def clean_up(ser):
@@ -113,14 +112,15 @@ def make_intermediate_lb():
         metric_stats[f"{metric}_display"] = metric_stats.apply(
             lambda row: f"{row[f'{metric}_mean']:.4f} ± {row[f'{metric}_std']:.4f}", axis=1
         )
-        cld_df = cld_df.merge(metric_stats[['user', f'{metric}_mean', f'{metric}_std', f'{metric}_display']], on='user', how='left')
+        cld_df = metric_stats[['user', f'{metric}_mean', f'{metric}_std', f'{metric}_display']].merge(cld_df, on='user', how='left')
 
     # re-sort by RAE mean, lowest is best
     cld_df = cld_df.sort_values(by='RAE_mean', ascending=True).reset_index(drop=True)
+    cld_df['user'] = cld_df['user'].str.split('___').str[1]
 
+    cld_subset = cld_df[['user', 'fixed_letter'] + [f'{metric}_display' for metric in METRICS]]
+    cld_subset = cld_subset.rename(columns={'user': 'user', 'fixed_letter': 'CLD'})
 
-    cld_subset = cld_df[['user_fixed', 'fixed_letter'] + [f'{metric}_display' for metric in METRICS]]
-    cld_subset = cld_subset.rename(columns={'user_fixed': 'user', 'fixed_letter': 'CLD'})
     print(cld_subset.head())
     cld_subset.to_csv("leaderboard_cld_results.csv", index=False)
 

utils.py

@@ -43,8 +43,10 @@ def check_page_exists(url: str, delay=0.2, max_retries=3, current_retries=0):
         # Check for Rate Limit Error and retry if under the limit
         if response.status_code == 429:
             if current_retries < max_retries:
-                print(f"Warning: Rate limit hit on {safe_url}. Attempt {current_retries + 1}/{max_retries}. Waiting for 5 seconds...")
-                time.sleep(5)
+                # Make wait time exponential
+                wait_time = 5 * (2 ** current_retries)
+                print(f"Warning: Rate limit hit on {safe_url}. Attempt {current_retries + 1}/{max_retries}. Waiting for {wait_time} seconds...")
+                time.sleep(wait_time)
                 # Recurse with an incremented retry counter
                 return check_page_exists(safe_url, delay=delay, max_retries=max_retries, current_retries=current_retries + 1) 
             else: