From 7c2bf8e1c7d324cbbf5495995d1d87b989bfffe9 Mon Sep 17 00:00:00 2001 From: scawful Date: Fri, 3 Oct 2025 12:47:15 -0400 Subject: [PATCH] Add ToolDispatcher for Enhanced Tool Call Management - Introduced `ToolDispatcher` class to handle tool calls from the AI agent, allowing for dynamic execution of commands based on user requests. - Updated `ConversationalAgentService` to integrate tool dispatching, enabling the agent to respond to tool calls and manage execution results. - Enhanced `AgentResponse` structure to include a list of tool calls, facilitating communication between the AI and the tool dispatcher. - Modified AI service implementations to parse and include tool calls in responses, improving the agent's interactive capabilities. This commit significantly enhances the z3ed system's ability to manage and execute tool calls, paving the way for more complex interactions in ROM hacking. --- src/app/app.cmake | 1 + .../agent/conversational_agent_service.cc | 15 +++++++ .../agent/conversational_agent_service.h | 2 + src/cli/service/agent/tool_dispatcher.cc | 40 +++++++++++++++++++ src/cli/service/agent/tool_dispatcher.h | 24 +++++++++++ src/cli/service/ai/common.h | 10 +++++ src/cli/service/ai/gemini_ai_service.cc | 17 ++++++++ src/cli/service/ai/ollama_ai_service.cc | 17 ++++++++ src/cli/service/ai/prompt_builder.cc | 29 +++++++++++++- src/cli/service/ai/prompt_builder.h | 1 + src/cli/z3ed.cmake | 2 + 11 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 src/cli/service/agent/tool_dispatcher.cc create mode 100644 src/cli/service/agent/tool_dispatcher.h diff --git a/src/app/app.cmake b/src/app/app.cmake index f929472c..cc72c10f 100644 --- a/src/app/app.cmake +++ b/src/app/app.cmake @@ -285,6 +285,7 @@ if(YAZE_WITH_GRPC) ${CMAKE_SOURCE_DIR}/src/cli/service/planning/tile16_proposal_generator.cc ${CMAKE_SOURCE_DIR}/src/cli/service/resources/resource_context_builder.cc ${CMAKE_SOURCE_DIR}/src/cli/service/resources/resource_catalog.cc + ${CMAKE_SOURCE_DIR}/src/cli/service/agent/tool_dispatcher.cc ) # Link gRPC libraries diff --git a/src/cli/service/agent/conversational_agent_service.cc b/src/cli/service/agent/conversational_agent_service.cc index 9cf03c65..a6c6be50 100644 --- a/src/cli/service/agent/conversational_agent_service.cc +++ b/src/cli/service/agent/conversational_agent_service.cc @@ -34,6 +34,21 @@ absl::StatusOr ConversationalAgentService::SendMessage( response_text += "\n\nCommands:\n" + absl::StrJoin(agent_response.commands, "\n"); } + // If the agent requested a tool call, dispatch it. + if (!agent_response.tool_calls.empty()) { + for (const auto& tool_call : agent_response.tool_calls) { + auto tool_result_or = tool_dispatcher_.Dispatch(tool_call); + if (tool_result_or.ok()) { + // Add the tool result to the history and send back to the AI. + history_.push_back({ChatMessage::Sender::kAgent, tool_result_or.value(), absl::Now()}); + return SendMessage(""); // Re-prompt the AI with the new context. + } else { + // Handle tool execution error. + return absl::InternalError(absl::StrCat("Tool execution failed: ", tool_result_or.status().message())); + } + } + } + ChatMessage chat_response = {ChatMessage::Sender::kAgent, response_text, absl::Now()}; diff --git a/src/cli/service/agent/conversational_agent_service.h b/src/cli/service/agent/conversational_agent_service.h index f5eabf23..0e155e66 100644 --- a/src/cli/service/agent/conversational_agent_service.h +++ b/src/cli/service/agent/conversational_agent_service.h @@ -6,6 +6,7 @@ #include "absl/status/statusor.h" #include "cli/service/ai/ai_service.h" +#include "cli/service/agent/tool_dispatcher.h" namespace yaze { namespace cli { @@ -31,6 +32,7 @@ class ConversationalAgentService { private: std::vector history_; std::unique_ptr ai_service_; + ToolDispatcher tool_dispatcher_; }; } // namespace agent diff --git a/src/cli/service/agent/tool_dispatcher.cc b/src/cli/service/agent/tool_dispatcher.cc new file mode 100644 index 00000000..11767037 --- /dev/null +++ b/src/cli/service/agent/tool_dispatcher.cc @@ -0,0 +1,40 @@ +#include "cli/service/agent/tool_dispatcher.h" + +#include "absl/strings/str_format.h" +#include "cli/handlers/agent/commands.h" + +namespace yaze { +namespace cli { +namespace agent { + +absl::StatusOr ToolDispatcher::Dispatch( + const ToolCall& tool_call) { + std::vector args; + for (const auto& [key, value] : tool_call.args) { + args.push_back(absl::StrFormat("--%s", key)); + args.push_back(value); + } + + if (tool_call.tool_name == "resource-list") { + // Note: This is a simplified approach for now. A more robust solution + // would capture stdout instead of relying on the handler to return a string. + auto status = HandleResourceListCommand(args); + if (!status.ok()) { + return status; + } + return "Successfully listed resources."; + } else if (tool_call.tool_name == "dungeon-list-sprites") { + auto status = HandleDungeonListSpritesCommand(args); + if (!status.ok()) { + return status; + } + return "Successfully listed sprites."; + } + + return absl::UnimplementedError( + absl::StrFormat("Unknown tool: %s", tool_call.tool_name)); +} + +} // namespace agent +} // namespace cli +} // namespace yaze diff --git a/src/cli/service/agent/tool_dispatcher.h b/src/cli/service/agent/tool_dispatcher.h new file mode 100644 index 00000000..ab4d8371 --- /dev/null +++ b/src/cli/service/agent/tool_dispatcher.h @@ -0,0 +1,24 @@ +#ifndef YAZE_SRC_CLI_SERVICE_AGENT_TOOL_DISPATCHER_H_ +#define YAZE_SRC_CLI_SERVICE_AGENT_TOOL_DISPATCHER_H_ + +#include +#include "absl/status/statusor.h" +#include "cli/service/ai/common.h" + +namespace yaze { +namespace cli { +namespace agent { + +class ToolDispatcher { + public: + ToolDispatcher() = default; + + // Execute a tool call and return the result as a string. + absl::StatusOr Dispatch(const ToolCall& tool_call); +}; + +} // namespace agent +} // namespace cli +} // namespace yaze + +#endif // YAZE_SRC_CLI_SERVICE_AGENT_TOOL_DISPATCHER_H_ diff --git a/src/cli/service/ai/common.h b/src/cli/service/ai/common.h index e26afe9c..ac6053a6 100644 --- a/src/cli/service/ai/common.h +++ b/src/cli/service/ai/common.h @@ -3,15 +3,25 @@ #include #include +#include namespace yaze { namespace cli { +// Represents a request from the AI to call a tool. +struct ToolCall { + std::string tool_name; + std::map args; +}; + // A structured response from an AI service. struct AgentResponse { // A natural language response to the user. std::string text_response; + // A list of tool calls the agent wants to make. + std::vector tool_calls; + // A list of z3ed commands to be executed. std::vector commands; diff --git a/src/cli/service/ai/gemini_ai_service.cc b/src/cli/service/ai/gemini_ai_service.cc index baec080c..0dd37494 100644 --- a/src/cli/service/ai/gemini_ai_service.cc +++ b/src/cli/service/ai/gemini_ai_service.cc @@ -207,6 +207,23 @@ absl::StatusOr GeminiAIService::ParseGeminiResponse( agent_response.reasoning = response_json["reasoning"].get(); } + if (response_json.contains("tool_calls") && + response_json["tool_calls"].is_array()) { + for (const auto& call : response_json["tool_calls"]) { + if (call.contains("tool_name") && call["tool_name"].is_string()) { + ToolCall tool_call; + tool_call.tool_name = call["tool_name"].get(); + if (call.contains("args") && call["args"].is_object()) { + for (auto& [key, value] : call["args"].items()) { + if (value.is_string()) { + tool_call.args[key] = value.get(); + } + } + } + agent_response.tool_calls.push_back(tool_call); + } + } + } if (response_json.contains("commands") && response_json["commands"].is_array()) { for (const auto& cmd : response_json["commands"]) { diff --git a/src/cli/service/ai/ollama_ai_service.cc b/src/cli/service/ai/ollama_ai_service.cc index bbf004f0..7922b9c1 100644 --- a/src/cli/service/ai/ollama_ai_service.cc +++ b/src/cli/service/ai/ollama_ai_service.cc @@ -246,6 +246,23 @@ absl::StatusOr OllamaAIService::GenerateResponse( response_json["reasoning"].is_string()) { agent_response.reasoning = response_json["reasoning"].get(); } + if (response_json.contains("tool_calls") && + response_json["tool_calls"].is_array()) { + for (const auto& call : response_json["tool_calls"]) { + if (call.contains("tool_name") && call["tool_name"].is_string()) { + ToolCall tool_call; + tool_call.tool_name = call["tool_name"].get(); + if (call.contains("args") && call["args"].is_object()) { + for (auto& [key, value] : call["args"].items()) { + if (value.is_string()) { + tool_call.args[key] = value.get(); + } + } + } + agent_response.tool_calls.push_back(tool_call); + } + } + } if (response_json.contains("commands") && response_json["commands"].is_array()) { for (const auto& cmd : response_json["commands"]) { diff --git a/src/cli/service/ai/prompt_builder.cc b/src/cli/service/ai/prompt_builder.cc index a6693251..de7e2446 100644 --- a/src/cli/service/ai/prompt_builder.cc +++ b/src/cli/service/ai/prompt_builder.cc @@ -122,6 +122,17 @@ void PromptBuilder::LoadDefaultExamples() { "Yes, I can validate the ROM for you.", {"rom validate"}, "Validation ensures ROM integrity after tile modifications"}); + + // ========================================================================== + // Q&A / Tool Use + // ========================================================================== + examples_.push_back( + {"What dungeons are in this project?", + "I can list the dungeons for you. Let me check the resource labels.", + {}, + "The user is asking a question. I need to use the `resource-list` tool " + "to find the answer.", + {{"resource-list", {{"type", "dungeon"}}}}}); } absl::Status PromptBuilder::LoadResourceCatalogue(const std::string& yaml_path) { @@ -197,6 +208,18 @@ std::string PromptBuilder::BuildFewShotExamplesSection() { oss << "**Commands:**\n"; oss << "```json\n{"; oss << " \"text_response\": \"" << example.text_response << "\",\n"; + oss << " \"tool_calls\": ["; + std::vector tool_calls; + for (const auto& call : example.tool_calls) { + std::vector args; + for (const auto& [key, value] : call.args) { + args.push_back("\"" + key + "\": \"" + value + "\""); + } + tool_calls.push_back("{\"tool_name\": \"" + call.tool_name + + "\", \"args\": {" + absl::StrJoin(args, ", ") + "}}"); + } + oss << absl::StrJoin(tool_calls, ", "); + oss << "],\n"; oss << " \"commands\": ["; std::vector quoted_cmds; @@ -220,12 +243,14 @@ std::string PromptBuilder::BuildConstraintsSection() { 1. **Output Format:** You MUST respond with ONLY a JSON object with the following structure: { "text_response": "Your natural language reply to the user.", + "tool_calls": [{ "tool_name": "tool_name", "args": { "arg1": "value1" } }], "commands": ["command1", "command2"], "reasoning": "Your thought process." } - `text_response` is for conversational replies. - - `commands` is for executable z3ed commands. It can be an empty array. - - NO explanatory text before or after the JSON object. + - `tool_calls` is for asking questions about the ROM. Use the available tools. + - `commands` is for generating commands to modify the ROM. + - All fields are optional. 2. **Command Syntax:** Follow the exact syntax shown in examples - Use correct flag names (--group, --id, --to, --from, etc.) diff --git a/src/cli/service/ai/prompt_builder.h b/src/cli/service/ai/prompt_builder.h index f996f4ff..3bf5cf74 100644 --- a/src/cli/service/ai/prompt_builder.h +++ b/src/cli/service/ai/prompt_builder.h @@ -23,6 +23,7 @@ struct FewShotExample { std::string text_response; std::vector expected_commands; std::string explanation; // Why these commands work + std::vector tool_calls; }; // ROM context information to inject into prompts diff --git a/src/cli/z3ed.cmake b/src/cli/z3ed.cmake index 46790222..c7ceb913 100644 --- a/src/cli/z3ed.cmake +++ b/src/cli/z3ed.cmake @@ -71,6 +71,8 @@ add_executable( cli/service/agent/conversational_agent_service.cc cli/service/ai/service_factory.h cli/service/ai/service_factory.cc + cli/service/agent/tool_dispatcher.h + cli/service/agent/tool_dispatcher.cc app/rom.cc app/core/project.cc app/core/asar_wrapper.cc