santimber commited on
Commit
2ca5952
ยท
1 Parent(s): d5e8077

improved tools

Browse files
Files changed (2) hide show
  1. app.py +44 -48
  2. tools.py +20 -0
app.py CHANGED
@@ -18,7 +18,8 @@ from tools import (
18
  video_analysis_tool,
19
  audio_processing_tool,
20
  file_type_detection_tool,
21
- read_file_tool
 
22
  )
23
  import re
24
 
@@ -30,6 +31,7 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
30
  llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
31
  tools = [
32
  serp_search_tool,
 
33
  download_file,
34
  image_recognition_tool,
35
  reverse_text_tool,
@@ -52,10 +54,6 @@ class MyAgent(TypedDict):
52
  # File Handling Functions
53
  # =========================
54
  def process_question_with_files(question_data: dict) -> str:
55
- """
56
- Process a question that may have attached files.
57
- Downloads and processes files, then combines with the question.
58
- """
59
  question_text = question_data.get('question', '')
60
  file_name = question_data.get('file_name', '')
61
 
@@ -64,44 +62,40 @@ def process_question_with_files(question_data: dict) -> str:
64
 
65
  print(f"๐Ÿ“Ž Processing question with attached file: {file_name}")
66
  try:
67
- # Download the file from the API
68
  file_url = f"{DEFAULT_API_URL}/files/{file_name}"
69
  local_file_path = f"/tmp/{file_name}"
70
- print(f"๐Ÿ“ฅ Downloading file from: {file_url}")
71
  download_result = download_file(file_url, local_file_path)
72
- print(f"๐Ÿ“ฅ Download result: {download_result}")
73
  if "Failed to download" in download_result:
74
  return f"{question_text}\n\n[Note: Could not download attached file {file_name}]"
75
- # Detect file type
76
- file_extension = file_name.lower().split('.')[-1]
77
- if file_extension in ['png', 'jpg', 'jpeg', 'gif', 'bmp']:
78
- print(f"๐Ÿ–ผ๏ธ Processing image file: {file_name}")
79
- image_result = image_recognition_tool.invoke(local_file_path)
80
- enhanced_question = f"{question_text}\n\n[Image Analysis: {image_result}]"
81
- elif file_extension in ['mp3', 'wav', 'm4a', 'flac', 'ogg']:
82
- print(f"๐ŸŽต Processing audio file: {file_name}")
83
- audio_result = audio_processing_tool.invoke(local_file_path)
84
- enhanced_question = f"{question_text}\n\n[Audio Transcription: {audio_result}]"
85
- elif file_extension in ['xls', 'xlsx', 'csv']:
86
- print(f"๐Ÿ“Š Processing spreadsheet file: {file_name}")
87
- file_content = read_file_tool.invoke(local_file_path)
88
- enhanced_question = f"{question_text}\n\n[Spreadsheet Content: {file_content}]"
89
- elif file_extension == 'py':
90
- print(f"๐Ÿ Processing Python file: {file_name}")
91
- code_content = read_file_tool.invoke(local_file_path)
92
- enhanced_question = f"{question_text}\n\n[Python Code: {code_content}]"
93
  else:
94
- print(f"๐Ÿ“„ Processing general file: {file_name}")
95
- file_content = read_file_tool.invoke(local_file_path)
96
- enhanced_question = f"{question_text}\n\n[File Content: {file_content}]"
 
97
  try:
98
  os.remove(local_file_path)
99
- print(f"๐Ÿงน Cleaned up temporary file: {local_file_path}")
100
- except:
101
  pass
102
- return enhanced_question
 
 
103
  except Exception as e:
104
- print(f"โŒ Error processing file {file_name}: {e}")
105
  return f"{question_text}\n\n[Note: Error processing attached file {file_name}: {str(e)}]"
106
 
107
 
@@ -173,27 +167,29 @@ DECISION MAKING:
173
  1. First, try to answer from your knowledge if it's a general fact.
174
  2. If you need specific, current, or detailed information, use serp_search_tool ONCE.
175
  3. If the question looks reversed (starts with a period), use reverse_text_tool ONCE first.
176
- 4. For file-based questions, use the appropriate file tool.
177
- 5. After using a tool, analyze the result and provide your final answer.
178
- 6. Do NOT cycle between tools unnecessarily.
 
179
 
180
  Tool Use Guidelines:
181
  1. Do **not** use any tools outside of the provided tools list.
182
  2. Always use **only one tool at a time** in each step of your execution.
183
  3. You have a MAXIMUM of 3 tool uses per question.
184
  4. For web searches and current information, use **serp_search_tool** (15s timeout).
185
- 5. If the question looks reversed (starts with a period or reads backward), first use **reverse_text_tool** to reverse it, then process the question.
186
- 6. For image analysis and description, use **image_recognition_tool** (requires OpenAI API key).
187
- 7. For Python code execution, use **python_execution_tool**.
188
- 8. For video analysis, use **video_analysis_tool**.
189
- 9. For audio processing, use **audio_processing_tool**.
190
- 10. For file type detection, use **file_type_detection_tool**.
191
- 11. For reading file contents, use **read_file_tool**.
192
- 12. File downloading is handled automatically - you don't need to download files manually.
193
- 13. Keep responses concise and efficient.
194
- 14. If you can't find the answer after using 2-3 tools, provide your best estimate based on available information.
195
- 15. NEVER use more than 3 tools for a single question.
196
- 16. After using a tool, provide your final answer immediately.
 
197
 
198
  FILE PROCESSING:
199
  - Questions may come with attached files (mp3, excel, images, etc.)
 
18
  video_analysis_tool,
19
  audio_processing_tool,
20
  file_type_detection_tool,
21
+ read_file_tool,
22
+ wiki_search_tool
23
  )
24
  import re
25
 
 
31
  llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
32
  tools = [
33
  serp_search_tool,
34
+ wiki_search_tool,
35
  download_file,
36
  image_recognition_tool,
37
  reverse_text_tool,
 
54
  # File Handling Functions
55
  # =========================
56
  def process_question_with_files(question_data: dict) -> str:
 
 
 
 
57
  question_text = question_data.get('question', '')
58
  file_name = question_data.get('file_name', '')
59
 
 
62
 
63
  print(f"๐Ÿ“Ž Processing question with attached file: {file_name}")
64
  try:
65
+ # Download the file
66
  file_url = f"{DEFAULT_API_URL}/files/{file_name}"
67
  local_file_path = f"/tmp/{file_name}"
 
68
  download_result = download_file(file_url, local_file_path)
 
69
  if "Failed to download" in download_result:
70
  return f"{question_text}\n\n[Note: Could not download attached file {file_name}]"
71
+
72
+ # Route based on file type
73
+ ext = file_name.lower().split('.')[-1]
74
+ if ext in ['mp3', 'wav', 'm4a', 'flac', 'ogg']:
75
+ result = audio_processing_tool.invoke(local_file_path)
76
+ tag = "Audio Transcription"
77
+ elif ext in ['png', 'jpg', 'jpeg', 'gif', 'bmp']:
78
+ result = image_recognition_tool.invoke(local_file_path)
79
+ tag = "Image Analysis"
80
+ elif ext in ['csv', 'xls', 'xlsx']:
81
+ result = read_file_tool.invoke(local_file_path)
82
+ tag = "Spreadsheet Content"
83
+ elif ext in ['txt', 'md', 'py', 'json']:
84
+ result = read_file_tool.invoke(local_file_path)
85
+ tag = "File Content"
 
 
 
86
  else:
87
+ result = read_file_tool.invoke(local_file_path)
88
+ tag = "File Content"
89
+
90
+ # Clean up
91
  try:
92
  os.remove(local_file_path)
93
+ except Exception:
 
94
  pass
95
+
96
+ return f"{question_text}\n\n[{tag}: {result}]"
97
+
98
  except Exception as e:
 
99
  return f"{question_text}\n\n[Note: Error processing attached file {file_name}: {str(e)}]"
100
 
101
 
 
167
  1. First, try to answer from your knowledge if it's a general fact.
168
  2. If you need specific, current, or detailed information, use serp_search_tool ONCE.
169
  3. If the question looks reversed (starts with a period), use reverse_text_tool ONCE first.
170
+ 4. For factual or historical questions, use wiki_search_tool ONCE.
171
+ 5. For file-based questions, use the appropriate file tool.
172
+ 6. After using a tool, analyze the result and provide your final answer.
173
+ 7. Do NOT cycle between tools unnecessarily.
174
 
175
  Tool Use Guidelines:
176
  1. Do **not** use any tools outside of the provided tools list.
177
  2. Always use **only one tool at a time** in each step of your execution.
178
  3. You have a MAXIMUM of 3 tool uses per question.
179
  4. For web searches and current information, use **serp_search_tool** (15s timeout).
180
+ 5. For factual or historical questions, use **wiki_search_tool**.
181
+ 6. If the question looks reversed (starts with a period or reads backward), first use **reverse_text_tool** to reverse it, then process the question.
182
+ 7. For image analysis and description, use **image_recognition_tool** (requires OpenAI API key).
183
+ 8. For Python code execution, use **python_execution_tool**.
184
+ 9. For video analysis, use **video_analysis_tool**.
185
+ 10. For audio processing, use **audio_processing_tool**.
186
+ 11. For file type detection, use **file_type_detection_tool**.
187
+ 12. For reading file contents, use **read_file_tool**.
188
+ 13. File downloading is handled automatically - you don't need to download files manually.
189
+ 14. Keep responses concise and efficient.
190
+ 15. If you can't find the answer after using 2-3 tools, provide your best estimate based on available information.
191
+ 16. NEVER use more than 3 tools for a single question.
192
+ 17. After using a tool, provide your final answer immediately.
193
 
194
  FILE PROCESSING:
195
  - Questions may come with attached files (mp3, excel, images, etc.)
tools.py CHANGED
@@ -20,6 +20,7 @@ import openai
20
  from pydub import AudioSegment
21
  import pandas as pd
22
  from PIL import Image
 
23
 
24
  # Load environment variables
25
  print("Current working directory:", os.getcwd())
@@ -526,3 +527,22 @@ math_calculation_tool = Tool(
526
  func=calculate_simple_math,
527
  description="Safely evaluates simple mathematical expressions. Use this when you need to perform basic math calculations."
528
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  from pydub import AudioSegment
21
  import pandas as pd
22
  from PIL import Image
23
+ from langchain_community.document_loaders import WikipediaLoader
24
 
25
  # Load environment variables
26
  print("Current working directory:", os.getcwd())
 
527
  func=calculate_simple_math,
528
  description="Safely evaluates simple mathematical expressions. Use this when you need to perform basic math calculations."
529
  )
530
+
531
+
532
+ def wiki_search(query: str) -> str:
533
+ """Search Wikipedia for a query and return maximum 2 results."""
534
+ search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
535
+ formatted_search_docs = "\n\n---\n\n".join(
536
+ [
537
+ f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>'
538
+ f"\n{doc.page_content}\n</Document>"
539
+ for doc in search_docs
540
+ ])
541
+ return formatted_search_docs
542
+
543
+
544
+ wiki_search_tool = Tool(
545
+ name="wiki_search_tool",
546
+ func=wiki_search,
547
+ description="Search Wikipedia for a query and return up to 2 results. Use this for factual or historical questions."
548
+ )