diff --git a/src/app/app.cmake b/src/app/app.cmake index f929472c..cc72c10f 100644 --- a/src/app/app.cmake +++ b/src/app/app.cmake @@ -285,6 +285,7 @@ if(YAZE_WITH_GRPC) ${CMAKE_SOURCE_DIR}/src/cli/service/planning/tile16_proposal_generator.cc ${CMAKE_SOURCE_DIR}/src/cli/service/resources/resource_context_builder.cc ${CMAKE_SOURCE_DIR}/src/cli/service/resources/resource_catalog.cc + ${CMAKE_SOURCE_DIR}/src/cli/service/agent/tool_dispatcher.cc ) # Link gRPC libraries diff --git a/src/cli/service/agent/conversational_agent_service.cc b/src/cli/service/agent/conversational_agent_service.cc index 9cf03c65..a6c6be50 100644 --- a/src/cli/service/agent/conversational_agent_service.cc +++ b/src/cli/service/agent/conversational_agent_service.cc @@ -34,6 +34,21 @@ absl::StatusOr ConversationalAgentService::SendMessage( response_text += "\n\nCommands:\n" + absl::StrJoin(agent_response.commands, "\n"); } + // If the agent requested a tool call, dispatch it. + if (!agent_response.tool_calls.empty()) { + for (const auto& tool_call : agent_response.tool_calls) { + auto tool_result_or = tool_dispatcher_.Dispatch(tool_call); + if (tool_result_or.ok()) { + // Add the tool result to the history and send back to the AI. + history_.push_back({ChatMessage::Sender::kAgent, tool_result_or.value(), absl::Now()}); + return SendMessage(""); // Re-prompt the AI with the new context. + } else { + // Handle tool execution error. + return absl::InternalError(absl::StrCat("Tool execution failed: ", tool_result_or.status().message())); + } + } + } + ChatMessage chat_response = {ChatMessage::Sender::kAgent, response_text, absl::Now()}; diff --git a/src/cli/service/agent/conversational_agent_service.h b/src/cli/service/agent/conversational_agent_service.h index f5eabf23..0e155e66 100644 --- a/src/cli/service/agent/conversational_agent_service.h +++ b/src/cli/service/agent/conversational_agent_service.h @@ -6,6 +6,7 @@ #include "absl/status/statusor.h" #include "cli/service/ai/ai_service.h" +#include "cli/service/agent/tool_dispatcher.h" namespace yaze { namespace cli { @@ -31,6 +32,7 @@ class ConversationalAgentService { private: std::vector history_; std::unique_ptr ai_service_; + ToolDispatcher tool_dispatcher_; }; } // namespace agent diff --git a/src/cli/service/agent/tool_dispatcher.cc b/src/cli/service/agent/tool_dispatcher.cc new file mode 100644 index 00000000..11767037 --- /dev/null +++ b/src/cli/service/agent/tool_dispatcher.cc @@ -0,0 +1,40 @@ +#include "cli/service/agent/tool_dispatcher.h" + +#include "absl/strings/str_format.h" +#include "cli/handlers/agent/commands.h" + +namespace yaze { +namespace cli { +namespace agent { + +absl::StatusOr ToolDispatcher::Dispatch( + const ToolCall& tool_call) { + std::vector args; + for (const auto& [key, value] : tool_call.args) { + args.push_back(absl::StrFormat("--%s", key)); + args.push_back(value); + } + + if (tool_call.tool_name == "resource-list") { + // Note: This is a simplified approach for now. A more robust solution + // would capture stdout instead of relying on the handler to return a string. + auto status = HandleResourceListCommand(args); + if (!status.ok()) { + return status; + } + return "Successfully listed resources."; + } else if (tool_call.tool_name == "dungeon-list-sprites") { + auto status = HandleDungeonListSpritesCommand(args); + if (!status.ok()) { + return status; + } + return "Successfully listed sprites."; + } + + return absl::UnimplementedError( + absl::StrFormat("Unknown tool: %s", tool_call.tool_name)); +} + +} // namespace agent +} // namespace cli +} // namespace yaze diff --git a/src/cli/service/agent/tool_dispatcher.h b/src/cli/service/agent/tool_dispatcher.h new file mode 100644 index 00000000..ab4d8371 --- /dev/null +++ b/src/cli/service/agent/tool_dispatcher.h @@ -0,0 +1,24 @@ +#ifndef YAZE_SRC_CLI_SERVICE_AGENT_TOOL_DISPATCHER_H_ +#define YAZE_SRC_CLI_SERVICE_AGENT_TOOL_DISPATCHER_H_ + +#include +#include "absl/status/statusor.h" +#include "cli/service/ai/common.h" + +namespace yaze { +namespace cli { +namespace agent { + +class ToolDispatcher { + public: + ToolDispatcher() = default; + + // Execute a tool call and return the result as a string. + absl::StatusOr Dispatch(const ToolCall& tool_call); +}; + +} // namespace agent +} // namespace cli +} // namespace yaze + +#endif // YAZE_SRC_CLI_SERVICE_AGENT_TOOL_DISPATCHER_H_ diff --git a/src/cli/service/ai/common.h b/src/cli/service/ai/common.h index e26afe9c..ac6053a6 100644 --- a/src/cli/service/ai/common.h +++ b/src/cli/service/ai/common.h @@ -3,15 +3,25 @@ #include #include +#include namespace yaze { namespace cli { +// Represents a request from the AI to call a tool. +struct ToolCall { + std::string tool_name; + std::map args; +}; + // A structured response from an AI service. struct AgentResponse { // A natural language response to the user. std::string text_response; + // A list of tool calls the agent wants to make. + std::vector tool_calls; + // A list of z3ed commands to be executed. std::vector commands; diff --git a/src/cli/service/ai/gemini_ai_service.cc b/src/cli/service/ai/gemini_ai_service.cc index baec080c..0dd37494 100644 --- a/src/cli/service/ai/gemini_ai_service.cc +++ b/src/cli/service/ai/gemini_ai_service.cc @@ -207,6 +207,23 @@ absl::StatusOr GeminiAIService::ParseGeminiResponse( agent_response.reasoning = response_json["reasoning"].get(); } + if (response_json.contains("tool_calls") && + response_json["tool_calls"].is_array()) { + for (const auto& call : response_json["tool_calls"]) { + if (call.contains("tool_name") && call["tool_name"].is_string()) { + ToolCall tool_call; + tool_call.tool_name = call["tool_name"].get(); + if (call.contains("args") && call["args"].is_object()) { + for (auto& [key, value] : call["args"].items()) { + if (value.is_string()) { + tool_call.args[key] = value.get(); + } + } + } + agent_response.tool_calls.push_back(tool_call); + } + } + } if (response_json.contains("commands") && response_json["commands"].is_array()) { for (const auto& cmd : response_json["commands"]) { diff --git a/src/cli/service/ai/ollama_ai_service.cc b/src/cli/service/ai/ollama_ai_service.cc index bbf004f0..7922b9c1 100644 --- a/src/cli/service/ai/ollama_ai_service.cc +++ b/src/cli/service/ai/ollama_ai_service.cc @@ -246,6 +246,23 @@ absl::StatusOr OllamaAIService::GenerateResponse( response_json["reasoning"].is_string()) { agent_response.reasoning = response_json["reasoning"].get(); } + if (response_json.contains("tool_calls") && + response_json["tool_calls"].is_array()) { + for (const auto& call : response_json["tool_calls"]) { + if (call.contains("tool_name") && call["tool_name"].is_string()) { + ToolCall tool_call; + tool_call.tool_name = call["tool_name"].get(); + if (call.contains("args") && call["args"].is_object()) { + for (auto& [key, value] : call["args"].items()) { + if (value.is_string()) { + tool_call.args[key] = value.get(); + } + } + } + agent_response.tool_calls.push_back(tool_call); + } + } + } if (response_json.contains("commands") && response_json["commands"].is_array()) { for (const auto& cmd : response_json["commands"]) { diff --git a/src/cli/service/ai/prompt_builder.cc b/src/cli/service/ai/prompt_builder.cc index a6693251..de7e2446 100644 --- a/src/cli/service/ai/prompt_builder.cc +++ b/src/cli/service/ai/prompt_builder.cc @@ -122,6 +122,17 @@ void PromptBuilder::LoadDefaultExamples() { "Yes, I can validate the ROM for you.", {"rom validate"}, "Validation ensures ROM integrity after tile modifications"}); + + // ========================================================================== + // Q&A / Tool Use + // ========================================================================== + examples_.push_back( + {"What dungeons are in this project?", + "I can list the dungeons for you. Let me check the resource labels.", + {}, + "The user is asking a question. I need to use the `resource-list` tool " + "to find the answer.", + {{"resource-list", {{"type", "dungeon"}}}}}); } absl::Status PromptBuilder::LoadResourceCatalogue(const std::string& yaml_path) { @@ -197,6 +208,18 @@ std::string PromptBuilder::BuildFewShotExamplesSection() { oss << "**Commands:**\n"; oss << "```json\n{"; oss << " \"text_response\": \"" << example.text_response << "\",\n"; + oss << " \"tool_calls\": ["; + std::vector tool_calls; + for (const auto& call : example.tool_calls) { + std::vector args; + for (const auto& [key, value] : call.args) { + args.push_back("\"" + key + "\": \"" + value + "\""); + } + tool_calls.push_back("{\"tool_name\": \"" + call.tool_name + + "\", \"args\": {" + absl::StrJoin(args, ", ") + "}}"); + } + oss << absl::StrJoin(tool_calls, ", "); + oss << "],\n"; oss << " \"commands\": ["; std::vector quoted_cmds; @@ -220,12 +243,14 @@ std::string PromptBuilder::BuildConstraintsSection() { 1. **Output Format:** You MUST respond with ONLY a JSON object with the following structure: { "text_response": "Your natural language reply to the user.", + "tool_calls": [{ "tool_name": "tool_name", "args": { "arg1": "value1" } }], "commands": ["command1", "command2"], "reasoning": "Your thought process." } - `text_response` is for conversational replies. - - `commands` is for executable z3ed commands. It can be an empty array. - - NO explanatory text before or after the JSON object. + - `tool_calls` is for asking questions about the ROM. Use the available tools. + - `commands` is for generating commands to modify the ROM. + - All fields are optional. 2. **Command Syntax:** Follow the exact syntax shown in examples - Use correct flag names (--group, --id, --to, --from, etc.) diff --git a/src/cli/service/ai/prompt_builder.h b/src/cli/service/ai/prompt_builder.h index f996f4ff..3bf5cf74 100644 --- a/src/cli/service/ai/prompt_builder.h +++ b/src/cli/service/ai/prompt_builder.h @@ -23,6 +23,7 @@ struct FewShotExample { std::string text_response; std::vector expected_commands; std::string explanation; // Why these commands work + std::vector tool_calls; }; // ROM context information to inject into prompts diff --git a/src/cli/z3ed.cmake b/src/cli/z3ed.cmake index 46790222..c7ceb913 100644 --- a/src/cli/z3ed.cmake +++ b/src/cli/z3ed.cmake @@ -71,6 +71,8 @@ add_executable( cli/service/agent/conversational_agent_service.cc cli/service/ai/service_factory.h cli/service/ai/service_factory.cc + cli/service/agent/tool_dispatcher.h + cli/service/agent/tool_dispatcher.cc app/rom.cc app/core/project.cc app/core/asar_wrapper.cc