add custom tools to query image and do ASR

.gitignore

@@ -0,0 +1 @@
+__pycache__

app.py

@@ -11,8 +11,9 @@ from llama_index.tools.wikipedia import WikipediaToolSpec
 from llama_index.tools.duckduckgo import DuckDuckGoSearchToolSpec
 from llama_index.tools.code_interpreter import CodeInterpreterToolSpec
 
-from prompt import custom_react_system_header_str
 from config import HF_MODEL_NAME, HF_PROVIDER
+from prompt import custom_react_system_header_str
+from custom_tools import query_image_tool, automatic_speech_recognition_tool
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -43,9 +44,10 @@ class LLamaIndexAgent:
 
         # tools definition
         tool_spec_list = []
-        # tool_spec_list += WikipediaToolSpec().to_tool_list()
+        tool_spec_list += WikipediaToolSpec().to_tool_list()
         tool_spec_list += DuckDuckGoSearchToolSpec().to_tool_list()
         tool_spec_list += CodeInterpreterToolSpec().to_tool_list()
+        tool_spec_list += [query_image_tool, automatic_speech_recognition_tool]
 
         # agent definition
         self.agent = ReActAgent(llm=llm, tools=tool_spec_list)
@@ -151,10 +153,23 @@ async def run_and_submit_all(profile: gr.OAuthProfile | None):
     for item in questions_data:
         task_id = item.get("task_id")
         question_text = item.get("question")
+        file_name = item.get("file_name")
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
+            if file_name:
+                # add the URL of the data source to the question (so that the agent can deal with it)
+                file_url = f"{DEFAULT_API_URL}/files/{task_id}"
+                question_text += f"\nFile URL: {file_url}"
+                # get the extension of the file to help the agent
+                try:
+                    ext = file_name.split('.')[-1]
+                    question_text += f" (.{ext} file)"
+                except:
+                    pass
+
+            # call the agent
             submitted_answer = await agent(question_text)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})

config.py

@@ -1,2 +1,8 @@
-HF_MODEL_NAME = "google/gemma-3-27b-it" # multimodal (need to handle image input)
-HF_PROVIDER = "nebius"
+HF_MODEL_NAME = "google/gemma-3-27b-it"
+HF_PROVIDER = "nebius"
+
+# HF_MODEL_NAME = "Qwen/Qwen2.5-Coder-32B-Instruct"
+# HF_PROVIDER = "hf-inference"
+
+# HF_MODEL_NAME = "Qwen/Qwen3-32B"
+# HF_PROVIDER = "hf-inference"

custom_tools.py

@@ -0,0 +1,63 @@
+# custom tools
+
+from huggingface_hub import InferenceClient
+from llama_index.core.tools import FunctionTool
+
+# --- Functions --- #
+
+def query_image(query: str, image_url: str) -> str:
+    """Ask anything about an image using a Vision Language Model
+
+    Args:
+        query (str): the query about the image, e.g. how many persons are on the image?
+        image_url (str): the URL to the image
+    """
+
+    client = InferenceClient(provider="nebius")
+    try:
+        completion = client.chat.completions.create(
+            # model="google/gemma-3-27b-it",
+            model="Qwen/Qwen2.5-VL-72B-Instruct",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": query
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": image_url
+                            }
+                        }
+                    ]
+                }
+            ],
+            max_tokens=512,
+        )
+        return completion.choices[0].message
+
+    except Exception as e:
+        return f"query_image failed: {e}"
+
+
+def automatic_speech_recognition(file_url: str) -> str:
+    """Transcribe an audio file to text
+
+    Args:
+        file_url (str): the URL to the audio file
+    """
+    client = InferenceClient(provider="fal-ai")
+    try:
+        return client.automatic_speech_recognition(file_url, model="openai/whisper-large-v3")
+    except Exception as e:
+        return f"automatic_speech_recognition failed: {e}"
+
+
+
+### --- Tool instance ---
+
+query_image_tool = FunctionTool.from_defaults(query_image)
+automatic_speech_recognition_tool = FunctionTool.from_defaults(automatic_speech_recognition)