Commit
Β·
a70ff2b
1
Parent(s):
adff627
docs: UI improvements
Browse files
app.py
CHANGED
|
@@ -1428,6 +1428,76 @@ def create_gradio_ui():
|
|
| 1428 |
|
| 1429 |
---
|
| 1430 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1431 |
## MCP Integration
|
| 1432 |
|
| 1433 |
This Gradio app is MCP-enabled. When deployed to HuggingFace Spaces, it can be accessed via MCP clients.
|
|
@@ -1476,77 +1546,93 @@ def create_gradio_ui():
|
|
| 1476 |
|
| 1477 |
with gr.Tab("βοΈ Settings"):
|
| 1478 |
gr.Markdown("""
|
| 1479 |
-
# βοΈ API
|
| 1480 |
|
| 1481 |
-
##
|
| 1482 |
|
| 1483 |
-
|
| 1484 |
|
| 1485 |
-
For
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1486 |
|
| 1487 |
-
##
|
| 1488 |
|
| 1489 |
-
|
|
|
|
|
|
|
|
|
|
| 1490 |
|
| 1491 |
-
|
| 1492 |
-
|
| 1493 |
-
|
|
|
|
| 1494 |
|
| 1495 |
-
|
| 1496 |
-
- Use any tool in the tabs above
|
| 1497 |
-
- Connect from TraceMind-AI (the MCP tools will use your keys)
|
| 1498 |
-
- Test with Claude Desktop (will use your keys)
|
| 1499 |
|
| 1500 |
-
## Security
|
| 1501 |
|
| 1502 |
-
β
**Session-only storage
|
| 1503 |
-
β
**No
|
| 1504 |
-
β
**
|
| 1505 |
-
β
**
|
| 1506 |
|
| 1507 |
---
|
| 1508 |
""")
|
| 1509 |
|
| 1510 |
-
# Show current key status (masked)
|
| 1511 |
current_gemini = os.environ.get("GEMINI_API_KEY", "")
|
| 1512 |
current_hf = os.environ.get("HF_TOKEN", "")
|
| 1513 |
|
| 1514 |
-
gemini_display =
|
| 1515 |
-
hf_display =
|
|
|
|
|
|
|
| 1516 |
|
| 1517 |
gr.Markdown(f"""
|
| 1518 |
-
|
| 1519 |
|
| 1520 |
-
|
| 1521 |
-
|
|
|
|
|
|
|
| 1522 |
|
| 1523 |
-
{
|
|
|
|
|
|
|
| 1524 |
""")
|
| 1525 |
|
| 1526 |
-
gr.Markdown("
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1527 |
|
| 1528 |
with gr.Row():
|
| 1529 |
with gr.Column():
|
| 1530 |
gemini_api_key_input = gr.Textbox(
|
| 1531 |
-
label="Google Gemini API Key",
|
| 1532 |
-
placeholder="Leave empty
|
| 1533 |
type="password",
|
| 1534 |
value="",
|
| 1535 |
-
info="
|
| 1536 |
)
|
| 1537 |
|
| 1538 |
with gr.Row():
|
| 1539 |
with gr.Column():
|
| 1540 |
hf_token_input = gr.Textbox(
|
| 1541 |
-
label="HuggingFace Token",
|
| 1542 |
-
placeholder="Leave empty
|
| 1543 |
type="password",
|
| 1544 |
value="",
|
| 1545 |
-
info="Get
|
| 1546 |
)
|
| 1547 |
|
| 1548 |
with gr.Row():
|
| 1549 |
-
save_keys_btn = gr.Button("πΎ
|
| 1550 |
reset_keys_btn = gr.Button("π Reset to Defaults", variant="secondary", size="lg")
|
| 1551 |
|
| 1552 |
settings_status = gr.Markdown("")
|
|
@@ -1554,66 +1640,67 @@ def create_gradio_ui():
|
|
| 1554 |
gr.Markdown("""
|
| 1555 |
---
|
| 1556 |
|
| 1557 |
-
|
|
|
|
|
|
|
|
|
|
| 1558 |
|
| 1559 |
-
|
| 1560 |
|
| 1561 |
-
1.
|
| 1562 |
-
2.
|
| 1563 |
-
3.
|
| 1564 |
-
4. Generate an API key
|
| 1565 |
-
5. Copy the key (starts with `AIza...`)
|
| 1566 |
|
| 1567 |
-
|
| 1568 |
|
| 1569 |
-
|
|
|
|
|
|
|
| 1570 |
|
| 1571 |
-
|
| 1572 |
-
2. Click "New token"
|
| 1573 |
-
3. Give it a name (e.g., "TraceMind Access")
|
| 1574 |
-
4. Select permissions:
|
| 1575 |
-
- **Read**: Sufficient for viewing datasets (leaderboard, traces, results)
|
| 1576 |
-
- **Write**: Required for `push_dataset_to_hub` tool (uploading synthetic datasets)
|
| 1577 |
-
5. Create and copy the token (starts with `hf_...`)
|
| 1578 |
|
| 1579 |
-
|
|
|
|
|
|
|
| 1580 |
""")
|
| 1581 |
|
| 1582 |
# Event handlers for Settings tab
|
| 1583 |
def save_override_keys(gemini, hf):
|
| 1584 |
"""Save user-provided API keys to session (override Spaces Secrets)"""
|
| 1585 |
-
|
| 1586 |
|
| 1587 |
if gemini and gemini.strip():
|
| 1588 |
if gemini.startswith("AIza"):
|
| 1589 |
os.environ["GEMINI_API_KEY"] = gemini.strip()
|
| 1590 |
-
|
| 1591 |
-
logger.info("Gemini API key overridden by user
|
| 1592 |
else:
|
| 1593 |
-
|
| 1594 |
|
| 1595 |
if hf and hf.strip():
|
| 1596 |
if hf.startswith("hf_"):
|
| 1597 |
os.environ["HF_TOKEN"] = hf.strip()
|
| 1598 |
-
|
| 1599 |
-
logger.info("HuggingFace token overridden by user
|
| 1600 |
else:
|
| 1601 |
-
|
| 1602 |
|
| 1603 |
-
if not
|
| 1604 |
-
|
| 1605 |
|
| 1606 |
-
|
| 1607 |
-
|
| 1608 |
-
|
| 1609 |
-
return "\n\n".join(messages)
|
| 1610 |
|
| 1611 |
def reset_to_defaults():
|
| 1612 |
"""Reset to Spaces Secrets (requires page refresh)"""
|
| 1613 |
return """
|
| 1614 |
-
βΉοΈ
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1615 |
|
| 1616 |
-
|
| 1617 |
"""
|
| 1618 |
|
| 1619 |
# Wire up buttons with api_name=False for security
|
|
|
|
| 1428 |
|
| 1429 |
---
|
| 1430 |
|
| 1431 |
+
### 10. analyze_results
|
| 1432 |
+
|
| 1433 |
+
**Description**: Analyzes detailed test results and provides optimization recommendations
|
| 1434 |
+
|
| 1435 |
+
**Parameters**:
|
| 1436 |
+
- `results_repo` (str, required): HuggingFace dataset containing results
|
| 1437 |
+
- Format: `username/smoltrace-results-model-timestamp`
|
| 1438 |
+
- Must contain "smoltrace-results-" prefix
|
| 1439 |
+
- `analysis_focus` (str): Focus area for analysis (default: "comprehensive")
|
| 1440 |
+
- Options: "failures", "performance", "cost", "comprehensive"
|
| 1441 |
+
- `max_rows` (int): Maximum test cases to analyze (default: 100, range: 10-500)
|
| 1442 |
+
|
| 1443 |
+
**Returns**: JSON object with AI analysis including:
|
| 1444 |
+
- Overall statistics (success rate, average duration, total cost)
|
| 1445 |
+
- Failure patterns and root causes
|
| 1446 |
+
- Performance bottlenecks in specific test cases
|
| 1447 |
+
- Cost optimization opportunities
|
| 1448 |
+
- Tool usage patterns
|
| 1449 |
+
- Task-specific insights (which types work well vs poorly)
|
| 1450 |
+
- Actionable optimization recommendations
|
| 1451 |
+
|
| 1452 |
+
**Use Case**:
|
| 1453 |
+
After running an evaluation, analyze the detailed test results to understand why certain tests are failing and get specific recommendations for improving success rate.
|
| 1454 |
+
|
| 1455 |
+
**Example**:
|
| 1456 |
+
```python
|
| 1457 |
+
result = analyze_results(
|
| 1458 |
+
results_repo="kshitij/smoltrace-results-gpt4-20251120",
|
| 1459 |
+
analysis_focus="failures",
|
| 1460 |
+
max_rows=100
|
| 1461 |
+
)
|
| 1462 |
+
```
|
| 1463 |
+
|
| 1464 |
+
---
|
| 1465 |
+
|
| 1466 |
+
### 11. generate_prompt_template
|
| 1467 |
+
|
| 1468 |
+
**Description**: Generate customized smolagents prompt template for a specific domain and tool set
|
| 1469 |
+
|
| 1470 |
+
**Parameters**:
|
| 1471 |
+
- `domain` (str, required): Domain for the prompt template
|
| 1472 |
+
- Examples: "finance", "healthcare", "customer_support", "e-commerce"
|
| 1473 |
+
- `tool_names` (str, required): Comma-separated list of tool names
|
| 1474 |
+
- Format: "tool1,tool2,tool3"
|
| 1475 |
+
- Example: "get_stock_price,calculate_roi,fetch_company_info"
|
| 1476 |
+
- `agent_type` (str): Agent type (default: "tool")
|
| 1477 |
+
- Options: "tool" (ToolCallingAgent), "code" (CodeAgent)
|
| 1478 |
+
|
| 1479 |
+
**Returns**: JSON object containing:
|
| 1480 |
+
- Customized YAML prompt template
|
| 1481 |
+
- Metadata (domain, tools, agent_type, timestamp)
|
| 1482 |
+
- Usage instructions
|
| 1483 |
+
|
| 1484 |
+
**Use Case**:
|
| 1485 |
+
When you generate synthetic datasets with `generate_synthetic_dataset`, use this tool to create a matching prompt template that agents can use during evaluation. This ensures your evaluation setup is complete and ready to run.
|
| 1486 |
+
|
| 1487 |
+
**Integration**:
|
| 1488 |
+
The generated prompt template can be included in your HuggingFace dataset card, making it easy for anyone to run evaluations with your dataset.
|
| 1489 |
+
|
| 1490 |
+
**Example**:
|
| 1491 |
+
```python
|
| 1492 |
+
result = generate_prompt_template(
|
| 1493 |
+
domain="customer_support",
|
| 1494 |
+
tool_names="search_knowledge_base,create_ticket,send_email,escalate_to_human",
|
| 1495 |
+
agent_type="tool"
|
| 1496 |
+
)
|
| 1497 |
+
```
|
| 1498 |
+
|
| 1499 |
+
---
|
| 1500 |
+
|
| 1501 |
## MCP Integration
|
| 1502 |
|
| 1503 |
This Gradio app is MCP-enabled. When deployed to HuggingFace Spaces, it can be accessed via MCP clients.
|
|
|
|
| 1546 |
|
| 1547 |
with gr.Tab("βοΈ Settings"):
|
| 1548 |
gr.Markdown("""
|
| 1549 |
+
# βοΈ API Configuration
|
| 1550 |
|
| 1551 |
+
## Overview
|
| 1552 |
|
| 1553 |
+
TraceMind MCP Server operates with **pre-configured API keys** from HuggingFace Spaces Secrets, enabling immediate use without additional setup.
|
| 1554 |
|
| 1555 |
+
**For Judges & Advanced Users**: Override default credentials with your own API keys to:
|
| 1556 |
+
- Avoid rate limits during extensive testing
|
| 1557 |
+
- Track usage independently
|
| 1558 |
+
- Ensure uninterrupted evaluation access
|
| 1559 |
+
|
| 1560 |
+
---
|
| 1561 |
|
| 1562 |
+
## Configuration Options
|
| 1563 |
|
| 1564 |
+
**Default Mode** (Recommended)
|
| 1565 |
+
- Uses HuggingFace Spaces Secrets
|
| 1566 |
+
- No configuration required
|
| 1567 |
+
- Suitable for most evaluations
|
| 1568 |
|
| 1569 |
+
**Custom Mode** (Optional)
|
| 1570 |
+
- Session-scoped overrides
|
| 1571 |
+
- Your keys take precedence
|
| 1572 |
+
- Active until browser refresh
|
| 1573 |
|
| 1574 |
+
---
|
|
|
|
|
|
|
|
|
|
| 1575 |
|
| 1576 |
+
## Security
|
| 1577 |
|
| 1578 |
+
- β
**Session-only storage** - Keys never persisted to disk
|
| 1579 |
+
- β
**No API exposure** - Configuration endpoint disabled (`api_name=False`)
|
| 1580 |
+
- β
**HTTPS encrypted** - All communications secured
|
| 1581 |
+
- β
**Privacy-first** - Key status displayed without revealing characters
|
| 1582 |
|
| 1583 |
---
|
| 1584 |
""")
|
| 1585 |
|
| 1586 |
+
# Show current key status (fully masked for security)
|
| 1587 |
current_gemini = os.environ.get("GEMINI_API_KEY", "")
|
| 1588 |
current_hf = os.environ.get("HF_TOKEN", "")
|
| 1589 |
|
| 1590 |
+
gemini_display = "β
Configured" if current_gemini else "β Not configured"
|
| 1591 |
+
hf_display = "β
Configured" if current_hf else "β Not configured"
|
| 1592 |
+
|
| 1593 |
+
status_badge = "β
**Active Configuration**: HuggingFace Spaces Secrets" if (current_gemini and current_hf) else "β οΈ **Status**: Incomplete configuration"
|
| 1594 |
|
| 1595 |
gr.Markdown(f"""
|
| 1596 |
+
## Current Status
|
| 1597 |
|
| 1598 |
+
| Component | Status |
|
| 1599 |
+
|-----------|--------|
|
| 1600 |
+
| **Google Gemini API** | {gemini_display} |
|
| 1601 |
+
| **HuggingFace Token** | {hf_display} |
|
| 1602 |
|
| 1603 |
+
{status_badge}
|
| 1604 |
+
|
| 1605 |
+
---
|
| 1606 |
""")
|
| 1607 |
|
| 1608 |
+
gr.Markdown("""
|
| 1609 |
+
## Custom API Keys
|
| 1610 |
+
|
| 1611 |
+
**Optional**: Override default credentials with your own keys for this session.
|
| 1612 |
+
""")
|
| 1613 |
|
| 1614 |
with gr.Row():
|
| 1615 |
with gr.Column():
|
| 1616 |
gemini_api_key_input = gr.Textbox(
|
| 1617 |
+
label="Google Gemini API Key (Optional)",
|
| 1618 |
+
placeholder="Leave empty for defaults or enter: AIza...",
|
| 1619 |
type="password",
|
| 1620 |
value="",
|
| 1621 |
+
info="Free tier: 1,500 requests/day β’ Get key: https://ai.google.dev/"
|
| 1622 |
)
|
| 1623 |
|
| 1624 |
with gr.Row():
|
| 1625 |
with gr.Column():
|
| 1626 |
hf_token_input = gr.Textbox(
|
| 1627 |
+
label="HuggingFace Token (Optional)",
|
| 1628 |
+
placeholder="Leave empty for defaults or enter: hf_...",
|
| 1629 |
type="password",
|
| 1630 |
value="",
|
| 1631 |
+
info="Permissions: Read (view) or Write (full access) β’ Get token: https://huggingface.co/settings/tokens"
|
| 1632 |
)
|
| 1633 |
|
| 1634 |
with gr.Row():
|
| 1635 |
+
save_keys_btn = gr.Button("πΎ Apply Configuration", variant="primary", size="lg")
|
| 1636 |
reset_keys_btn = gr.Button("π Reset to Defaults", variant="secondary", size="lg")
|
| 1637 |
|
| 1638 |
settings_status = gr.Markdown("")
|
|
|
|
| 1640 |
gr.Markdown("""
|
| 1641 |
---
|
| 1642 |
|
| 1643 |
+
## API Key Setup Guide
|
| 1644 |
+
|
| 1645 |
+
<details>
|
| 1646 |
+
<summary><b>π Click to expand setup instructions</b></summary>
|
| 1647 |
|
| 1648 |
+
### Google Gemini API
|
| 1649 |
|
| 1650 |
+
1. Visit [Google AI Studio](https://ai.google.dev/)
|
| 1651 |
+
2. Generate API key (starts with `AIza`)
|
| 1652 |
+
3. **Free tier**: 1,500 requests/day
|
|
|
|
|
|
|
| 1653 |
|
| 1654 |
+
### HuggingFace Token
|
| 1655 |
|
| 1656 |
+
1. Visit [HuggingFace Tokens](https://huggingface.co/settings/tokens)
|
| 1657 |
+
2. Create token with **Read** (view datasets) or **Write** (upload datasets) permissions
|
| 1658 |
+
3. Token starts with `hf_`
|
| 1659 |
|
| 1660 |
+
</details>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1661 |
|
| 1662 |
+
---
|
| 1663 |
+
|
| 1664 |
+
**Note**: Custom keys are session-scoped and cleared on page refresh.
|
| 1665 |
""")
|
| 1666 |
|
| 1667 |
# Event handlers for Settings tab
|
| 1668 |
def save_override_keys(gemini, hf):
|
| 1669 |
"""Save user-provided API keys to session (override Spaces Secrets)"""
|
| 1670 |
+
results = []
|
| 1671 |
|
| 1672 |
if gemini and gemini.strip():
|
| 1673 |
if gemini.startswith("AIza"):
|
| 1674 |
os.environ["GEMINI_API_KEY"] = gemini.strip()
|
| 1675 |
+
results.append("β
**Gemini API**: Configuration applied successfully")
|
| 1676 |
+
logger.info("Gemini API key overridden by user")
|
| 1677 |
else:
|
| 1678 |
+
results.append("β **Gemini API**: Invalid format (must start with 'AIza')")
|
| 1679 |
|
| 1680 |
if hf and hf.strip():
|
| 1681 |
if hf.startswith("hf_"):
|
| 1682 |
os.environ["HF_TOKEN"] = hf.strip()
|
| 1683 |
+
results.append("β
**HuggingFace Token**: Configuration applied successfully")
|
| 1684 |
+
logger.info("HuggingFace token overridden by user")
|
| 1685 |
else:
|
| 1686 |
+
results.append("β **HuggingFace Token**: Invalid format (must start with 'hf_')")
|
| 1687 |
|
| 1688 |
+
if not results:
|
| 1689 |
+
return "βΉοΈ **No changes**: Empty fields submitted. Default configuration remains active."
|
| 1690 |
|
| 1691 |
+
results.append("\n**Status**: Custom configuration active for this session.")
|
| 1692 |
+
return "\n\n".join(results)
|
|
|
|
|
|
|
| 1693 |
|
| 1694 |
def reset_to_defaults():
|
| 1695 |
"""Reset to Spaces Secrets (requires page refresh)"""
|
| 1696 |
return """
|
| 1697 |
+
βΉοΈ **Reset Instructions**
|
| 1698 |
+
|
| 1699 |
+
To restore default HuggingFace Spaces configuration:
|
| 1700 |
+
1. Refresh this page (F5 or Ctrl+R)
|
| 1701 |
+
2. Session overrides will be cleared automatically
|
| 1702 |
|
| 1703 |
+
Default credentials will be active after refresh.
|
| 1704 |
"""
|
| 1705 |
|
| 1706 |
# Wire up buttons with api_name=False for security
|