Refreshing less

src/backend/run_eval_suite_harness.py

@@ -35,11 +35,8 @@ def run_evaluation(eval_request: EvalRequest, task_names: list, num_fewshot: int
             "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
         )
 
-    try:
-        all_tasks = tasks.ALL_TASKS
-    except AttributeError:
-        task_manager = TaskManager()
-        all_tasks = task_manager.all_tasks
+    task_manager = TaskManager()
+    all_tasks = task_manager.all_tasks
         
     task_names = utils.pattern_match(task_names, all_tasks)
 

src/envs.py

@@ -9,10 +9,10 @@ TOKEN = os.environ.get("TOKEN") # A read/write token for your org
 OWNER = "meg" # Change to your org - don't forget to create a results and request dataset
 
 # For harness evaluations
-DEVICE = "cpu" # "cuda:0" if you add compute, for harness evaluations
+DEVICE = "cuda:0" #if you add compute, for harness evaluations
 LIMIT = 20 # !!!! For testing, should be None for actual evaluations!!!
 NUM_FEWSHOT = 0 # Change with your few shot for the Harness evaluations
-TASKS_HARNESS = ["realtoxicityprompts", "logiqa"]
+TASKS_HARNESS = ["realtoxicityprompts", "toxigen", "logiqa"]
 
 # For lighteval evaluations
 ACCELERATOR = "cpu"
@@ -35,7 +35,7 @@ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
-REFRESH_RATE = 10 * 60  # 10 min
+REFRESH_RATE = 30 * 60  # 30 min
 NUM_LINES_VISUALIZE = 300
 
 API = HfApi(token=TOKEN)