|
|
"""Tests for app handler functions. |
|
|
|
|
|
This module tests the main handler functions used in the Gradio UI, |
|
|
particularly focusing on input collection and execution handling. |
|
|
""" |
|
|
|
|
|
import json |
|
|
from unittest.mock import patch |
|
|
|
|
|
import pytest |
|
|
|
|
|
|
|
|
from app import handle_execute_plan |
|
|
from kg_services.ontology import MCPPrompt, MCPTool, PlannedStep |
|
|
|
|
|
|
|
|
class TestHandleExecutePlan: |
|
|
"""Test suite for handle_execute_plan function.""" |
|
|
|
|
|
@pytest.fixture |
|
|
def sample_tool(self) -> MCPTool: |
|
|
"""Create a sample MCPTool for testing.""" |
|
|
return MCPTool( |
|
|
tool_id="test-tool-001", |
|
|
name="Test Sentiment Analyzer", |
|
|
description="A test tool for sentiment analysis", |
|
|
tags=["sentiment", "analysis", "test"], |
|
|
invocation_command_stub="sentiment_analyze --input {text}", |
|
|
) |
|
|
|
|
|
@pytest.fixture |
|
|
def sample_prompt(self) -> MCPPrompt: |
|
|
"""Create a sample MCPPrompt for testing.""" |
|
|
return MCPPrompt( |
|
|
prompt_id="test-prompt-001", |
|
|
name="Basic Sentiment Analysis", |
|
|
description="Analyze sentiment of provided text", |
|
|
target_tool_id="test-tool-001", |
|
|
template_string="Analyze the sentiment of this text: {{text_input}}", |
|
|
input_variables=["text_input"], |
|
|
difficulty_level="beginner", |
|
|
example_inputs={"text_input": "This is a great product!"}, |
|
|
) |
|
|
|
|
|
@pytest.fixture |
|
|
def sample_planned_step( |
|
|
self, sample_tool: MCPTool, sample_prompt: MCPPrompt |
|
|
) -> PlannedStep: |
|
|
"""Create a sample PlannedStep for testing.""" |
|
|
return PlannedStep(tool=sample_tool, prompt=sample_prompt, relevance_score=0.95) |
|
|
|
|
|
@pytest.fixture |
|
|
def multi_input_prompt(self) -> MCPPrompt: |
|
|
"""Create a prompt with multiple input variables.""" |
|
|
return MCPPrompt( |
|
|
prompt_id="test-prompt-multi", |
|
|
name="Multi-Input Analysis", |
|
|
description="Analysis with multiple inputs", |
|
|
target_tool_id="test-tool-001", |
|
|
template_string="Analyze {{input1}} with context {{input2}} and format {{input3}}", |
|
|
input_variables=["input1", "input2", "input3"], |
|
|
difficulty_level="intermediate", |
|
|
) |
|
|
|
|
|
@pytest.fixture |
|
|
def no_input_prompt(self) -> MCPPrompt: |
|
|
"""Create a prompt with no input variables.""" |
|
|
return MCPPrompt( |
|
|
prompt_id="test-prompt-no-input", |
|
|
name="No Input Required", |
|
|
description="A prompt that requires no inputs", |
|
|
target_tool_id="test-tool-001", |
|
|
template_string="Perform standard analysis", |
|
|
input_variables=[], |
|
|
difficulty_level="beginner", |
|
|
) |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_basic_success( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test successful input collection with single variable.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_test_123", |
|
|
"tool_information": {"tool_name": "Test Sentiment Analyzer"}, |
|
|
"prompt_information": {"prompt_name": "Basic Sentiment Analysis"}, |
|
|
"execution_details": {"inputs_count": 1, "execution_time_ms": 1250}, |
|
|
"results": { |
|
|
"mock_output": "Mock analysis results", |
|
|
"confidence_score": 0.95, |
|
|
}, |
|
|
"metadata": {"simulation_version": "MVP3_Sprint4"}, |
|
|
} |
|
|
test_query = "analyze sentiment" |
|
|
test_input = "This is a great product!" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
assert "π **Execution Complete!**" in result |
|
|
assert "Test Sentiment Analyzer" in result |
|
|
assert "Basic Sentiment Analysis" in result |
|
|
assert test_input in result |
|
|
assert "β
" in result |
|
|
|
|
|
|
|
|
mock_planner_agent.generate_plan.assert_called_once_with(test_query, top_k=1) |
|
|
|
|
|
mock_executor_agent.execute_plan_step.assert_called_once() |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_multiple_inputs( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_tool, multi_input_prompt |
|
|
): |
|
|
"""Test input collection with multiple variables.""" |
|
|
|
|
|
planned_step = PlannedStep( |
|
|
tool=sample_tool, prompt=multi_input_prompt, relevance_score=0.88 |
|
|
) |
|
|
mock_planner_agent.generate_plan.return_value = [planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_test_456", |
|
|
"tool_information": {"tool_name": "Test Tool"}, |
|
|
"prompt_information": {"prompt_name": "Multi Input Prompt"}, |
|
|
"execution_details": {"inputs_count": 3, "execution_time_ms": 1250}, |
|
|
"results": { |
|
|
"mock_output": "Mock analysis results", |
|
|
"confidence_score": 0.95, |
|
|
}, |
|
|
"metadata": {"simulation_version": "MVP3_Sprint4"}, |
|
|
} |
|
|
|
|
|
test_query = "complex analysis" |
|
|
input1, input2, input3 = "data1", "context2", "format3" |
|
|
|
|
|
|
|
|
result = handle_execute_plan( |
|
|
test_query, input1, input2, input3, "unused4", "unused5" |
|
|
) |
|
|
|
|
|
|
|
|
assert "π **Execution Complete!**" in result |
|
|
assert input1 in result |
|
|
assert input2 in result |
|
|
assert input3 in result |
|
|
|
|
|
|
|
|
json_start = result.find("```json\n") |
|
|
if json_start != -1: |
|
|
json_start += 8 |
|
|
json_end = result.find("\n```", json_start) |
|
|
if json_end != -1: |
|
|
json_content = result[json_start:json_end].strip() |
|
|
if json_content: |
|
|
parsed_inputs = json.loads(json_content) |
|
|
assert parsed_inputs == {"input1": input1, "input2": input2, "input3": input3} |
|
|
else: |
|
|
|
|
|
assert all(inp in result for inp in [input1, input2, input3]) |
|
|
else: |
|
|
|
|
|
assert all(inp in result for inp in [input1, input2, input3]) |
|
|
else: |
|
|
|
|
|
assert all(inp in result for inp in [input1, input2, input3]) |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_no_inputs_required( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_tool, no_input_prompt |
|
|
): |
|
|
"""Test handling of prompts that require no inputs.""" |
|
|
|
|
|
planned_step = PlannedStep( |
|
|
tool=sample_tool, prompt=no_input_prompt, relevance_score=0.90 |
|
|
) |
|
|
mock_planner_agent.generate_plan.return_value = [planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_test_789", |
|
|
"tool_information": {"tool_name": "Test Tool"}, |
|
|
"prompt_information": {"prompt_name": "No Input Prompt"}, |
|
|
"execution_details": {"inputs_count": 0, "execution_time_ms": 1250}, |
|
|
"results": { |
|
|
"mock_output": "Mock analysis results", |
|
|
"confidence_score": 0.95, |
|
|
}, |
|
|
"metadata": {"simulation_version": "MVP3_Sprint4"}, |
|
|
} |
|
|
|
|
|
test_query = "standard analysis" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, "unused1", "unused2") |
|
|
|
|
|
|
|
|
assert "π **Execution Complete!**" in result |
|
|
assert "No Input Required" in result |
|
|
|
|
|
|
|
|
json_start = result.find("```json\n") |
|
|
if json_start != -1: |
|
|
json_start += 8 |
|
|
json_end = result.find("\n```", json_start) |
|
|
if json_end != -1: |
|
|
json_content = result[json_start:json_end].strip() |
|
|
if json_content: |
|
|
parsed_inputs = json.loads(json_content) |
|
|
assert parsed_inputs == {} |
|
|
|
|
|
|
|
|
def test_handle_execute_plan_no_planner_agent(self): |
|
|
"""Test error handling when planner agent is not available.""" |
|
|
|
|
|
with patch("app.planner_agent", None): |
|
|
result = handle_execute_plan("test query", "input1") |
|
|
|
|
|
|
|
|
assert "β **Error**: Planner service not available" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_empty_query(self, mock_planner_agent, mock_executor_agent): |
|
|
"""Test error handling with empty query.""" |
|
|
|
|
|
result = handle_execute_plan("", "input1") |
|
|
|
|
|
|
|
|
assert "β **Error**: Original query is missing" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_no_planned_steps(self, mock_planner_agent, mock_executor_agent): |
|
|
"""Test error handling when no planned steps are returned.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [] |
|
|
|
|
|
|
|
|
result = handle_execute_plan("test query", "input1") |
|
|
|
|
|
|
|
|
assert "β **Error**: Could not retrieve the current action plan" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_planner_exception(self, mock_planner_agent, mock_executor_agent): |
|
|
"""Test error handling when planner raises an exception.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.side_effect = Exception("Planner error") |
|
|
|
|
|
|
|
|
result = handle_execute_plan("test query", "input1") |
|
|
|
|
|
|
|
|
assert "β **Execution Error**" in result |
|
|
assert "Planner error" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_partial_inputs( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_tool, multi_input_prompt |
|
|
): |
|
|
"""Test handling when fewer inputs are provided than required.""" |
|
|
|
|
|
planned_step = PlannedStep( |
|
|
tool=sample_tool, prompt=multi_input_prompt, relevance_score=0.75 |
|
|
) |
|
|
mock_planner_agent.generate_plan.return_value = [planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_test_partial", |
|
|
"tool_information": {"tool_name": "Test Tool"}, |
|
|
"prompt_information": {"prompt_name": "Multi Input Prompt"}, |
|
|
"execution_details": {"inputs_count": 2, "execution_time_ms": 1250}, |
|
|
"results": { |
|
|
"mock_output": "Mock analysis results", |
|
|
"confidence_score": 0.95, |
|
|
}, |
|
|
"metadata": {"simulation_version": "MVP3_Sprint4"}, |
|
|
} |
|
|
|
|
|
test_query = "partial input test" |
|
|
|
|
|
input1, input2 = "data1", "context2" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, input1, input2) |
|
|
|
|
|
|
|
|
assert "π **Execution Complete!**" in result |
|
|
|
|
|
|
|
|
json_start = result.find("```json\n") |
|
|
if json_start != -1: |
|
|
json_start += 8 |
|
|
json_end = result.find("\n```", json_start) |
|
|
if json_end != -1: |
|
|
json_content = result[json_start:json_end].strip() |
|
|
if json_content: |
|
|
parsed_inputs = json.loads(json_content) |
|
|
|
|
|
expected = {"input1": input1, "input2": input2, "input3": ""} |
|
|
assert parsed_inputs == expected |
|
|
else: |
|
|
|
|
|
assert input1 in result and input2 in result |
|
|
else: |
|
|
|
|
|
assert input1 in result and input2 in result |
|
|
else: |
|
|
|
|
|
assert input1 in result and input2 in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
@patch("app.logger") |
|
|
def test_handle_execute_plan_logging( |
|
|
self, mock_logger, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test that appropriate logging occurs during execution.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_test_log", |
|
|
"tool_information": {"tool_name": "Test Sentiment Analyzer"}, |
|
|
"prompt_information": {"prompt_name": "Basic Sentiment Analysis"}, |
|
|
"execution_details": {"inputs_count": 1, "execution_time_ms": 1250}, |
|
|
"results": { |
|
|
"mock_output": "Mock analysis results", |
|
|
"confidence_score": 0.95, |
|
|
}, |
|
|
"metadata": {"simulation_version": "MVP3_Sprint4"}, |
|
|
} |
|
|
test_query = "logging test" |
|
|
test_input = "test input" |
|
|
|
|
|
|
|
|
handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
|
|
|
assert mock_logger.info.call_count >= 2 |
|
|
|
|
|
|
|
|
log_calls = [call[0][0] for call in mock_logger.info.call_args_list] |
|
|
assert any("Executing plan with StubExecutorAgent" in log for log in log_calls) |
|
|
assert any( |
|
|
"Execution simulation completed successfully" in log for log in log_calls |
|
|
) |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_json_formatting( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test that JSON formatting in output is valid.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_test_json", |
|
|
"tool_information": {"tool_name": "Test Sentiment Analyzer"}, |
|
|
"prompt_information": {"prompt_name": "Basic Sentiment Analysis"}, |
|
|
"execution_details": {"inputs_count": 1, "execution_time_ms": 1250}, |
|
|
"results": { |
|
|
"mock_output": "Mock analysis results", |
|
|
"confidence_score": 0.95, |
|
|
}, |
|
|
"metadata": {"simulation_version": "MVP3_Sprint4"}, |
|
|
} |
|
|
test_query = "json formatting test" |
|
|
test_input = "test input with special chars: @#$%^&*()" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
|
|
|
assert "π **Execution Complete!**" in result |
|
|
assert test_input in result |
|
|
assert "Mock analysis results" in result |
|
|
|
|
|
|
|
|
assert "@#$%^&*()" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_markdown_formatting( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test that execution results are properly formatted in Markdown.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_test_md", |
|
|
"tool_information": {"tool_name": "Test Sentiment Analyzer"}, |
|
|
"prompt_information": {"prompt_name": "Basic Sentiment Analysis"}, |
|
|
"execution_details": {"inputs_count": 1, "execution_time_ms": 1250}, |
|
|
"results": { |
|
|
"mock_output": "Mock analysis results", |
|
|
"confidence_score": 0.95, |
|
|
}, |
|
|
"metadata": {"simulation_version": "MVP3_Sprint4"}, |
|
|
} |
|
|
test_query = "markdown test" |
|
|
test_input = "test input" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
|
|
|
assert "# π **Execution Complete!**" in result |
|
|
assert "## π **Execution Summary**" in result |
|
|
assert "## π§ **Tool & Prompt Information**" in result |
|
|
assert "## π **Input Summary**" in result |
|
|
assert "## π― **Execution Results**" in result |
|
|
|
|
|
|
|
|
assert "- **Status**: β
" in result |
|
|
assert "- **Tool**:" in result |
|
|
assert "Mock analysis results" in result |
|
|
|
|
|
|
|
|
class TestErrorHandling: |
|
|
"""Test suite for error handling in app handlers.""" |
|
|
|
|
|
@pytest.fixture |
|
|
def sample_tool(self) -> MCPTool: |
|
|
"""Create a sample MCPTool for testing.""" |
|
|
return MCPTool( |
|
|
tool_id="test-tool-error", |
|
|
name="Test Error Tool", |
|
|
description="A tool for testing error scenarios", |
|
|
tags=["test", "error"], |
|
|
invocation_command_stub="test_error {input}", |
|
|
) |
|
|
|
|
|
@pytest.fixture |
|
|
def sample_prompt(self) -> MCPPrompt: |
|
|
"""Create a sample MCPPrompt for testing.""" |
|
|
return MCPPrompt( |
|
|
prompt_id="test-prompt-error", |
|
|
name="Test Error Prompt", |
|
|
description="A prompt for testing error scenarios", |
|
|
target_tool_id="test-tool-error", |
|
|
template_string="Process: {{text_input}}", |
|
|
input_variables=["text_input"], |
|
|
difficulty_level="beginner", |
|
|
) |
|
|
|
|
|
@pytest.fixture |
|
|
def sample_planned_step( |
|
|
self, sample_tool: MCPTool, sample_prompt: MCPPrompt |
|
|
) -> PlannedStep: |
|
|
"""Create a sample PlannedStep for testing.""" |
|
|
return PlannedStep(tool=sample_tool, prompt=sample_prompt, relevance_score=0.95) |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_error_response( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test handling of error responses from executor.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_error", |
|
|
"execution_id": "exec_err_test_123", |
|
|
"tool_information": {"tool_name": "Test Error Tool"}, |
|
|
"prompt_information": {"prompt_name": "Test Error Prompt"}, |
|
|
"execution_details": { |
|
|
"inputs_count": 1, |
|
|
"execution_time_ms": 500, |
|
|
"error_occurred_at": 45, |
|
|
}, |
|
|
"error_information": { |
|
|
"error_type": "user_requested", |
|
|
"error_code": "USR_REQ_001", |
|
|
"error_severity": "medium", |
|
|
"error_message": "User explicitly requested error simulation", |
|
|
"error_details": "Error simulation was triggered by user input containing error keywords", |
|
|
"suggested_fixes": [ |
|
|
"Remove error keywords from input", |
|
|
"Use different test data", |
|
|
], |
|
|
"retry_recommended": True, |
|
|
}, |
|
|
"results": { |
|
|
"message": "Tool execution failed: User explicitly requested error simulation", |
|
|
"mock_output": "**Error Simulation Activated**\n\nβ **User-Requested Error**\nThis error was triggered because your input contained error simulation keywords.", |
|
|
"confidence_score": 0.0, |
|
|
}, |
|
|
"metadata": { |
|
|
"simulation_version": "MVP3_Sprint4", |
|
|
"error_simulation": "user_requested", |
|
|
"trigger_info": "This should fail and show an error", |
|
|
}, |
|
|
} |
|
|
test_query = "error test" |
|
|
test_input = "This should fail and show an error" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
assert "# β **Execution Failed!**" in result |
|
|
assert "## π¨ **Error Summary**" in result |
|
|
assert "- **Status**: β Simulated Error" in result |
|
|
assert "- **Type**: user requested" in result |
|
|
assert "## β οΈ **Error Details**" in result |
|
|
assert "## π‘ **Recovery Suggestions**" in result |
|
|
assert "Remove error keywords from input" in result |
|
|
assert "Use different test data" in result |
|
|
assert "- **Retry Recommended**: β
Yes" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_security_error( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test handling of security violation errors.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_error", |
|
|
"execution_id": "exec_err_sec_456", |
|
|
"tool_information": {"tool_name": "Test Error Tool"}, |
|
|
"prompt_information": {"prompt_name": "Test Error Prompt"}, |
|
|
"execution_details": { |
|
|
"inputs_count": 1, |
|
|
"execution_time_ms": 250, |
|
|
"error_occurred_at": 15, |
|
|
}, |
|
|
"error_information": { |
|
|
"error_type": "security_violation", |
|
|
"error_code": "SEC_001", |
|
|
"error_severity": "high", |
|
|
"error_message": "Potential security violation detected", |
|
|
"error_details": "Input contains suspicious content that may pose security risks", |
|
|
"suggested_fixes": [ |
|
|
"Remove suspicious code/scripts from input", |
|
|
"Use plain text input only", |
|
|
], |
|
|
"retry_recommended": False, |
|
|
}, |
|
|
"results": { |
|
|
"message": "Tool execution failed: Potential security violation detected", |
|
|
"mock_output": "**Security Error**\n\nπ‘οΈ **Security Violation Detected**\nThe input contains content that may pose security risks.", |
|
|
"confidence_score": 0.0, |
|
|
}, |
|
|
"metadata": { |
|
|
"simulation_version": "MVP3_Sprint4", |
|
|
"error_simulation": "security_violation", |
|
|
"trigger_info": "suspicious_content", |
|
|
}, |
|
|
} |
|
|
test_query = "security test" |
|
|
test_input = "Process this <script>alert('hack')</script> content" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
assert "# β **Execution Failed!**" in result |
|
|
assert "- **Type**: security violation" in result |
|
|
assert "- **Retry Recommended**: β No" in result |
|
|
assert "Security Violation Detected" in result |
|
|
assert "Remove suspicious code/scripts from input" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_network_timeout_error( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test handling of network timeout errors.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_error", |
|
|
"execution_id": "exec_err_net_789", |
|
|
"tool_information": {"tool_name": "Test Error Tool"}, |
|
|
"prompt_information": {"prompt_name": "Test Error Prompt"}, |
|
|
"execution_details": { |
|
|
"inputs_count": 1, |
|
|
"execution_time_ms": 800, |
|
|
"error_occurred_at": 75, |
|
|
}, |
|
|
"error_information": { |
|
|
"error_type": "network_timeout", |
|
|
"error_code": "NET_001", |
|
|
"error_severity": "medium", |
|
|
"error_message": "Network request timed out", |
|
|
"error_details": "The tool failed to complete processing within the allowed time limit", |
|
|
"suggested_fixes": [ |
|
|
"Retry the operation", |
|
|
"Check network connectivity", |
|
|
"Try with smaller input", |
|
|
], |
|
|
"retry_recommended": True, |
|
|
}, |
|
|
"results": { |
|
|
"message": "Tool execution failed: Network request timed out", |
|
|
"mock_output": "**Network Timeout**\n\nβ° **Operation Timed Out**\nThe request took too long to process and was automatically cancelled.", |
|
|
"confidence_score": 0.0, |
|
|
}, |
|
|
"metadata": { |
|
|
"simulation_version": "MVP3_Sprint4", |
|
|
"error_simulation": "network_timeout", |
|
|
"trigger_info": "random", |
|
|
}, |
|
|
} |
|
|
test_query = "network timeout test" |
|
|
test_input = "normal input" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
assert "# β **Execution Failed!**" in result |
|
|
assert "- **Type**: network timeout" in result |
|
|
assert "Operation Timed Out" in result |
|
|
assert "Retry the operation" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_error_markdown_structure( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test that error responses maintain proper Markdown structure.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_error", |
|
|
"execution_id": "exec_err_md_test", |
|
|
"tool_information": {"tool_name": "Test Error Tool"}, |
|
|
"prompt_information": {"prompt_name": "Test Error Prompt"}, |
|
|
"execution_details": { |
|
|
"inputs_count": 1, |
|
|
"execution_time_ms": 300, |
|
|
"error_occurred_at": 30, |
|
|
}, |
|
|
"error_information": { |
|
|
"error_type": "input_too_large", |
|
|
"error_code": "VAL_001", |
|
|
"error_severity": "medium", |
|
|
"error_message": "Input exceeds maximum allowed size", |
|
|
"error_details": "Input size limit exceeded", |
|
|
"suggested_fixes": [ |
|
|
"Reduce input size", |
|
|
"Split large inputs into smaller chunks", |
|
|
], |
|
|
"retry_recommended": True, |
|
|
}, |
|
|
"results": { |
|
|
"message": "Tool execution failed: Input exceeds maximum allowed size", |
|
|
"mock_output": "**Input Size Error**\n\nπ **Input Too Large**\nThe provided input exceeds the maximum allowed size for this tool.", |
|
|
"confidence_score": 0.0, |
|
|
}, |
|
|
"metadata": { |
|
|
"simulation_version": "MVP3_Sprint4", |
|
|
"error_simulation": "input_too_large", |
|
|
"trigger_info": "large_input", |
|
|
}, |
|
|
} |
|
|
test_query = "markdown structure test" |
|
|
test_input = "test input" |
|
|
|
|
|
|
|
|
result = handle_execute_plan(test_query, test_input) |
|
|
|
|
|
|
|
|
assert result.startswith("# β **Execution Failed!**") |
|
|
assert "## π¨ **Error Summary**" in result |
|
|
assert "## π§ **Tool & Prompt Information**" in result |
|
|
assert "## β οΈ **Error Details**" in result |
|
|
assert "## π‘ **Recovery Suggestions**" in result |
|
|
assert "## π **Input Summary**" in result |
|
|
assert "## π **Error Output**" in result |
|
|
|
|
|
|
|
|
assert "1. Reduce input size" in result |
|
|
assert "2. Split large inputs into smaller chunks" in result |
|
|
|
|
|
|
|
|
assert "---" in result |
|
|
|
|
|
@patch("app.executor_agent") |
|
|
@patch("app.planner_agent") |
|
|
def test_handle_execute_plan_error_vs_success_distinction( |
|
|
self, mock_planner_agent, mock_executor_agent, sample_planned_step |
|
|
): |
|
|
"""Test that error and success responses are clearly distinguished.""" |
|
|
|
|
|
mock_planner_agent.generate_plan.return_value = [sample_planned_step] |
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_error", |
|
|
"execution_id": "exec_err_test", |
|
|
"tool_information": {"tool_name": "Test Tool"}, |
|
|
"prompt_information": {"prompt_name": "Test Prompt"}, |
|
|
"execution_details": {"inputs_count": 1, "execution_time_ms": 300}, |
|
|
"error_information": { |
|
|
"error_type": "test_scenario", |
|
|
"error_code": "TST_ERR_001", |
|
|
"error_severity": "high", |
|
|
"error_message": "Test error scenario activated", |
|
|
"error_details": "Test error details", |
|
|
"suggested_fixes": ["This is for testing"], |
|
|
"retry_recommended": True, |
|
|
}, |
|
|
"results": {"mock_output": "Error output", "confidence_score": 0.0}, |
|
|
"metadata": { |
|
|
"simulation_version": "MVP3_Sprint4", |
|
|
"error_simulation": "test_scenario", |
|
|
}, |
|
|
} |
|
|
|
|
|
|
|
|
error_result = handle_execute_plan("error test", "test error") |
|
|
|
|
|
|
|
|
assert "β **Execution Failed!**" in error_result |
|
|
assert "π¨ **Error Summary**" in error_result |
|
|
assert "β οΈ **Error Details**" in error_result |
|
|
assert "π **Error Output**" in error_result |
|
|
assert "π **Execution Complete!**" not in error_result |
|
|
assert ( |
|
|
"- **Status**: β
" not in error_result |
|
|
) |
|
|
|
|
|
|
|
|
mock_executor_agent.execute_plan_step.return_value = { |
|
|
"status": "simulated_success", |
|
|
"execution_id": "exec_success_test", |
|
|
"tool_information": {"tool_name": "Test Tool"}, |
|
|
"prompt_information": {"prompt_name": "Test Prompt"}, |
|
|
"execution_details": {"inputs_count": 1, "execution_time_ms": 1500}, |
|
|
"results": {"mock_output": "Success output", "confidence_score": 0.95}, |
|
|
"metadata": { |
|
|
"simulation_version": "MVP3_Sprint4", |
|
|
"error_simulation": "none", |
|
|
}, |
|
|
} |
|
|
|
|
|
|
|
|
success_result = handle_execute_plan("success test", "normal input") |
|
|
|
|
|
|
|
|
assert "π **Execution Complete!**" in success_result |
|
|
assert "π **Execution Summary**" in success_result |
|
|
assert "π― **Execution Results**" in success_result |
|
|
assert "β **Execution Failed!**" not in success_result |
|
|
assert "π¨ **Error Summary**" not in success_result |
|
|
|