Change to docker space

Dockerfile

@@ -0,0 +1,19 @@
+# Dockerfile to run CRS Arena
+FROM python:3.9-bullseye
+
+COPY . .
+
+# Install requirements
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose Hugging Face Space secrets to environment variables
+RUN --mount=type=secret,id=models_folder_url,mode=0444,required=true echo "MODELS_FOLDER_URL=$(cat /run/secrets/models_folder_url)" >> .env
+RUN --mount=type=secret,id=item_embeddings_url,mode=0444,required=true echo "ITEM_EMBEDDINGS_URL=$(cat /run/secrets/item_embeddings_url)" >> .env
+
+# Download external data
+RUN python download_external_data.py
+
+EXPOSE 7860
+
+# Run Streamlit app
+CMD ["python", "-m", "streamlit", "run", "crs_arena.arena", "--server.port", "7860"]

README.md

@@ -3,11 +3,10 @@ title: CRSArena
 emoji: 🐠
 colorFrom: yellow
 colorTo: yellow
-sdk: streamlit
-sdk_version: 1.42.2
-app_file: crs_arena/arena.py
+sdk: docker
+app_port: 7860
 pinned: false
 license: mit
 ---
 
-Check out the configuration reference at <https://huggingface.co/docs/hub/spaces-config-reference>
+Check out the configuration reference at <https://huggingface.co/docs/hub/spaces-config-reference>

crs_arena/arena.py

@@ -22,8 +22,6 @@ import asyncio
 import json
 import logging
 import os
-import threading
-import time
 from copy import deepcopy
 from datetime import datetime
 from typing import Dict, List
@@ -37,12 +35,7 @@ from battle_manager import (
 )
 from crs_fighter import CRSFighter
 from streamlit_lottie import st_lottie_spinner
-from utils import (
-    download_and_extract_item_embeddings,
-    download_and_extract_models,
-    upload_conversation_logs_to_hf,
-    upload_feedback_to_gsheet,
-)
+from utils import upload_conversation_logs_to_hf, upload_feedback_to_gsheet
 
 from src.model.crb_crs.recommender import *
 
@@ -56,14 +49,6 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
-# Download models and data externally stored if not already downloaded
-if not os.path.exists("data/models"):
-    logger.info("Downloading models...")
-    download_and_extract_models()
-if not os.path.exists("data/embed_items"):
-    logger.info("Downloading item embeddings...")
-    download_and_extract_item_embeddings()
-
 # Create the conversation logs directory
 CONVERSATION_LOG_DIR = "data/arena/conversation_logs/"
 os.makedirs(CONVERSATION_LOG_DIR, exist_ok=True)
@@ -82,7 +67,9 @@ def record_vote(vote: str) -> None:
     crs1_model: CRSFighter = st.session_state["crs1"]
     crs2_model: CRSFighter = st.session_state["crs2"]
     last_row_id = str(datetime.now())
-    logger.info(f"Vote: {last_row_id}, {user_id}, {crs1_model.name}, {crs2_model.name}, {vote}")
+    logger.info(
+        f"Vote: {last_row_id}, {user_id}, {crs1_model.name}, {crs2_model.name}, {vote}"
+    )
     asyncio.run(
         upload_feedback_to_gsheet(
             {
@@ -189,6 +176,7 @@ def get_crs_response(crs: CRSFighter, message: str) -> str:
     #    time.sleep(0.05)
     return response
 
+
 @st.dialog("Your vote has been submitted! Thank you!")
 def feedback_dialog(row_id: int) -> None:
     """Pop-up dialog to provide feedback after voting.
@@ -208,6 +196,7 @@ def feedback_dialog(row_id: int) -> None:
         st.session_state.clear()
         st.rerun()
 
+
 @st.fragment
 def chat_col(crs_id: int, color: str):
     """Chat column for the CRS model.

crs_arena/utils.py

@@ -4,16 +4,13 @@ import ast
 import asyncio
 import logging
 import os
-import sqlite3
 import sys
-import tarfile
 from datetime import timedelta
-from typing import Any, Dict, List
+from typing import Dict
 
 import openai
 import pandas as pd
 import streamlit as st
-import wget
 import yaml
 from huggingface_hub import HfApi
 from streamlit_gsheets.gsheets_connection import GSheetsServiceAccountClient
@@ -23,7 +20,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 from src.model.crs_model import CRSModel
 
 # Initialize Hugging Face API
-HF_API = HfApi(token=st.secrets["hf_token"])
+HF_API = HfApi(token=os.environ.get("hf_token"))
 
 
 @st.cache_resource(
@@ -52,52 +49,12 @@ def get_crs_model(model_name: str, model_config_file: str) -> CRSModel:
     model_args = yaml.safe_load(open(model_config_file, "r"))
 
     if "chatgpt" in model_name:
-        openai.api_key = st.secrets["openai_api_key"]
+        openai.api_key = os.environ.get("openai_api_key")
 
     # Extract crs model from name
     name = model_name.split("_")[0]
 
-    return CRSModel(name, **model_args), ttl=timedelta(days=3)
-)
-
-
-def download_and_extract_models() -> None:
-    """Downloads the models folder from the server and extracts it."""
-    logging.debug("Downloading models folder.")
-    models_url = st.secrets["models_folder_url"]
-    models_targz = "models.tar.gz"
-    models_folder = "data/models/"
-    try:
-        wget.download(models_url, models_targz)
-
-        logging.debug("Extracting models folder.")
-        with tarfile.open(models_targz, "r:gz") as tar:
-            tar.extractall(models_folder)
-
-        os.remove(models_targz)
-        logging.debug("Models folder downloaded and extracted.")
-    except Exception as e:
-        logging.error(f"Error downloading models folder: {e}")
-
-
-def download_and_extract_item_embeddings() -> None:
-    """Downloads the item embeddings folder from the server and extracts it."""
-    logging.debug("Downloading item embeddings folder.")
-    item_embeddings_url = st.secrets["item_embeddings_url"]
-    item_embeddings_tarbz = "item_embeddings.tar.bz2"
-    item_embeddings_folder = "data/"
-
-    try:
-        wget.download(item_embeddings_url, item_embeddings_tarbz)
-
-        logging.debug("Extracting item embeddings folder.")
-        with tarfile.open(item_embeddings_tarbz, "r:bz2") as tar:
-            tar.extractall(item_embeddings_folder)
-
-        os.remove(item_embeddings_tarbz)
-        logging.debug("Item embeddings folder downloaded and extracted.")
-    except Exception as e:
-        logging.error(f"Error downloading item embeddings folder: {e}")
+    return CRSModel(name, **model_args)
 
 
 async def upload_conversation_logs_to_hf(
@@ -122,7 +79,7 @@ async def upload_conversation_logs_to_hf(
             lambda: HF_API.upload_file(
                 path_or_fileobj=conversation_log_file_path,
                 path_in_repo=repo_filename,
-                repo_id=st.secrets["dataset_repo"],
+                repo_id=os.environ.get("dataset_repo"),
                 repo_type="dataset",
             ),
         )
@@ -164,7 +121,7 @@ def _upload_feedback_to_gsheet_sync(
         worksheet: Name of the worksheet to upload the feedback to.
     """
     gs_connection = GSheetsServiceAccountClient(
-        ast.literal_eval(st.secrets["gsheet"])
+        ast.literal_eval(os.environ.get("gsheet"))
     )
     df = gs_connection.read(worksheet=worksheet)
     if df[df["id"] == row["id"]].empty:

download_external_data.py

@@ -0,0 +1,56 @@
+"""Script to download external data for the project at build time."""
+
+import logging
+import os
+import tarfile
+
+import wget
+
+
+def download_and_extract_models() -> None:
+    """Downloads the models folder from the server and extracts it."""
+    logging.debug("Downloading models folder.")
+    models_url = os.environ.get("MODELS_FOLDER_URL")
+    models_targz = "models.tar.gz"
+    models_folder = "data/models/"
+    try:
+        wget.download(models_url, models_targz)
+
+        logging.debug("Extracting models folder.")
+        with tarfile.open(models_targz, "r:gz") as tar:
+            tar.extractall(models_folder)
+
+        os.remove(models_targz)
+        logging.debug("Models folder downloaded and extracted.")
+    except Exception as e:
+        logging.error(f"Error downloading models folder: {e}")
+
+
+def download_and_extract_item_embeddings() -> None:
+    """Downloads the item embeddings folder from the server and extracts it."""
+    logging.debug("Downloading item embeddings folder.")
+    item_embeddings_url = os.environ.get("ITEM_EMBEDDINGS_URL")
+    item_embeddings_tarbz = "item_embeddings.tar.bz2"
+    item_embeddings_folder = "data/"
+
+    try:
+        wget.download(item_embeddings_url, item_embeddings_tarbz)
+
+        logging.debug("Extracting item embeddings folder.")
+        with tarfile.open(item_embeddings_tarbz, "r:bz2") as tar:
+            tar.extractall(item_embeddings_folder)
+
+        os.remove(item_embeddings_tarbz)
+        logging.debug("Item embeddings folder downloaded and extracted.")
+    except Exception as e:
+        logging.error(f"Error downloading item embeddings folder: {e}")
+
+
+if __name__ == "__main__":
+    if not os.path.exists("data/models"):
+        logging.info("Downloading models...")
+        download_and_extract_models()
+
+    if not os.path.exists("data/embed_items"):
+        logging.info("Downloading item embeddings...")
+        download_and_extract_item_embeddings()

requirements.txt

@@ -10,7 +10,7 @@ tiktoken==0.7.0
 tenacity<9.0.0
 thefuzz==0.22.1
 numpy<2
-streamlit==1.42.2
+streamlit==1.38.0
 SQLAlchemy==1.4.0
 sent2vec==0.3.0
 wget==3.2