scawful/yaze
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance AI agent capabilities with new tool calling instructions, improved response handling, and terminal color utilities

Browse Source
This commit is contained in:
scawful
2025-10-04 03:04:22 -04:00
parent 06dcffb6ac
commit 2931634837
10 changed files with 562 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/cli/cli_main.cc
View File

@@ -16,6 +16,10 @@
ABSL_FLAG(bool, tui, false, "Launch Text User Interface");
ABSL_DECLARE_FLAG(std::string, rom);
ABSL_DECLARE_FLAG(std::string, ai_provider);
ABSL_DECLARE_FLAG(std::string, ai_model);
ABSL_DECLARE_FLAG(std::string, gemini_api_key);
ABSL_DECLARE_FLAG(std::string, ollama_host);
namespace {
@@ -75,6 +79,59 @@ ParsedGlobals ParseGlobalFlags(int argc, char* argv[]) {
absl::SetFlag(&FLAGS_rom, std::string(argv[++i]));
continue;
}
// AI provider flags
if (absl::StartsWith(token, "--ai_provider=")) {
absl::SetFlag(&FLAGS_ai_provider, std::string(token.substr(14)));
continue;
}
if (token == "--ai_provider") {
if (i + 1 >= argc) {
result.error = "--ai_provider flag requires a value";
return result;
}
absl::SetFlag(&FLAGS_ai_provider, std::string(argv[++i]));
continue;
}
if (absl::StartsWith(token, "--ai_model=")) {
absl::SetFlag(&FLAGS_ai_model, std::string(token.substr(11)));
continue;
}
if (token == "--ai_model") {
if (i + 1 >= argc) {
result.error = "--ai_model flag requires a value";
return result;
}
absl::SetFlag(&FLAGS_ai_model, std::string(argv[++i]));
continue;
}
if (absl::StartsWith(token, "--gemini_api_key=")) {
absl::SetFlag(&FLAGS_gemini_api_key, std::string(token.substr(17)));
continue;
}
if (token == "--gemini_api_key") {
if (i + 1 >= argc) {
result.error = "--gemini_api_key flag requires a value";
return result;
}
absl::SetFlag(&FLAGS_gemini_api_key, std::string(argv[++i]));
continue;
}
if (absl::StartsWith(token, "--ollama_host=")) {
absl::SetFlag(&FLAGS_ollama_host, std::string(token.substr(14)));
continue;
}
if (token == "--ollama_host") {
if (i + 1 >= argc) {
result.error = "--ollama_host flag requires a value";
return result;
}
absl::SetFlag(&FLAGS_ollama_host, std::string(argv[++i]));
continue;
}
}
result.positional.push_back(current);

56
src/cli/handlers/agent.cc
View File

@@ -12,8 +12,60 @@ namespace agent {
namespace {
constexpr absl::string_view kUsage =
"Usage: agent <run|plan|diff|accept|test|test-conversation|gui|learn|list|commit|revert|describe|resource-list|dungeon-list-sprites|overworld-find-tile|overworld-describe-map|overworld-list-warps|chat|simple-chat> "
"[options]";
"Usage: agent <subcommand> [options]\n"
"\n"
"AI-Powered Agent Subcommands:\n"
" simple-chat Simple text-based chat (recommended for testing)\n"
" Modes: interactive | piped | batch | single-message\n"
" Example: agent simple-chat \"What dungeons exist?\" --rom=zelda3.sfc\n"
" Example: agent simple-chat --rom=zelda3.sfc --ai_provider=ollama\n"
" Example: echo \"List sprites\" | agent simple-chat --rom=zelda3.sfc\n"
" Example: agent simple-chat --file=queries.txt --rom=zelda3.sfc\n"
"\n"
" test-conversation Run automated test conversation with AI\n"
" Example: agent test-conversation --rom=zelda3.sfc --ai_provider=ollama\n"
"\n"
" chat Full FTXUI-based chat interface\n"
" Example: agent chat --rom=zelda3.sfc\n"
"\n"
"ROM Inspection Tools (can be called by AI or directly):\n"
" resource-list List labeled resources (dungeons, sprites, etc.)\n"
" Example: agent resource-list --type=dungeon --format=json\n"
"\n"
" dungeon-list-sprites List sprites in a dungeon room\n"
" Example: agent dungeon-list-sprites --room=5 --format=json\n"
"\n"
" overworld-find-tile Search for tile placements in overworld\n"
" Example: agent overworld-find-tile --tile=0x02E --format=json\n"
"\n"
" overworld-describe-map Get metadata about an overworld map\n"
" Example: agent overworld-describe-map --map=0 --format=json\n"
"\n"
" overworld-list-warps List entrances/exits/holes in overworld\n"
" Example: agent overworld-list-warps --map=0 --format=json\n"
"\n"
"Proposal & Testing Commands:\n"
" run Execute agent task\n"
" plan Generate execution plan\n"
" diff Show ROM differences\n"
" accept Accept and apply proposal changes\n"
" test Run agent tests\n"
" gui Launch GUI components\n"
" learn Train agent on examples\n"
" list List available resources\n"
" commit Commit changes\n"
" revert Revert changes\n"
" describe Describe agent capabilities\n"
"\n"
"Global Options:\n"
" --rom=<path> Path to Zelda3 ROM file (required for most commands)\n"
" --ai_provider=<name> AI provider: mock (default) | ollama | gemini\n"
" --ai_model=<name> Model name (e.g., qwen2.5-coder:7b for Ollama)\n"
" --ollama_host=<url> Ollama server URL (default: http://localhost:11434)\n"
" --gemini_api_key=<key> Gemini API key (or set GEMINI_API_KEY env var)\n"
" --format=<type> Output format: json | table | yaml\n"
"\n"
"For more details, see: docs/simple_chat_input_methods.md";
} // namespace
} // namespace agent

50
src/cli/service/agent/conversational_agent_service.cc
View File

@@ -11,6 +11,7 @@
#include "absl/strings/str_join.h"
#include "absl/time/clock.h"
#include "cli/service/ai/service_factory.h"
#include "cli/util/terminal_colors.h"
#include "nlohmann/json.hpp"
namespace yaze {
@@ -174,9 +175,23 @@ absl::StatusOr<ChatMessage> ConversationalAgentService::SendMessage(
}
constexpr int kMaxToolIterations = 4;
bool waiting_for_text_response = false;
for (int iteration = 0; iteration < kMaxToolIterations; ++iteration) {
// Show loading indicator while waiting for AI response
util::LoadingIndicator loader(
waiting_for_text_response
? "Generating final response..."
: "Thinking...",
true);
loader.Start();
auto response_or = ai_service_->GenerateResponse(history_);
loader.Stop();
if (!response_or.ok()) {
util::PrintError(absl::StrCat(
"Failed to get AI response: ", response_or.status().message()));
return absl::InternalError(absl::StrCat(
"Failed to get AI response: ", response_or.status().message()));
}
@@ -184,28 +199,61 @@ absl::StatusOr<ChatMessage> ConversationalAgentService::SendMessage(
const auto& agent_response = response_or.value();
if (!agent_response.tool_calls.empty()) {
// Check if we were waiting for a text response but got more tool calls instead
if (waiting_for_text_response) {
util::PrintWarning(
absl::StrCat("LLM called tools again instead of providing final response (Iteration: ",
iteration, "/", kMaxToolIterations, ")"));
}
bool executed_tool = false;
for (const auto& tool_call : agent_response.tool_calls) {
// Format tool arguments for display
std::vector<std::string> arg_parts;
for (const auto& [key, value] : tool_call.args) {
arg_parts.push_back(absl::StrCat(key, "=", value));
}
std::string args_str = absl::StrJoin(arg_parts, ", ");
util::PrintToolCall(tool_call.tool_name, args_str);
auto tool_result_or = tool_dispatcher_.Dispatch(tool_call);
if (!tool_result_or.ok()) {
util::PrintError(absl::StrCat(
"Tool execution failed: ", tool_result_or.status().message()));
return absl::InternalError(absl::StrCat(
"Tool execution failed: ", tool_result_or.status().message()));
}
const std::string& tool_output = tool_result_or.value();
if (!tool_output.empty()) {
util::PrintSuccess("Tool executed successfully");
// Add tool result with a clear marker for the LLM
std::string marked_output = "[TOOL RESULT] " + tool_output;
history_.push_back(
CreateMessage(ChatMessage::Sender::kAgent, tool_output));
CreateMessage(ChatMessage::Sender::kUser, marked_output));
}
executed_tool = true;
}
if (executed_tool) {
// Now we're waiting for the LLM to provide a text response
waiting_for_text_response = true;
// Re-query the AI with updated context.
continue;
}
}
// Check if we received a text response after tool execution
if (waiting_for_text_response && agent_response.text_response.empty() &&
agent_response.commands.empty()) {
util::PrintWarning(
absl::StrCat("LLM did not provide text_response after receiving tool results (Iteration: ",
iteration, "/", kMaxToolIterations, ")"));
// Continue to give it another chance
continue;
}
std::string response_text = agent_response.text_response;
if (!agent_response.reasoning.empty()) {
if (!response_text.empty()) {

3
src/cli/service/ai/ai_service.cc
View File

@@ -110,8 +110,7 @@ absl::StatusOr<AgentResponse> MockAIService::GenerateResponse(
}
response.text_response =
"I'm not sure how to help with that yet. Try asking for resource labels "
"or listing dungeon sprites.";
"I'm just a mock service. Please load a provider like ollama or gemini.";
return response;
}

7
src/cli/service/ai/gemini_ai_service.cc
View File

@@ -348,9 +348,12 @@ absl::StatusOr<AgentResponse> GeminiAIService::ParseGeminiResponse(
absl::StrCat("❌ Failed to parse Gemini response: ", e.what()));
}
if (agent_response.commands.empty()) {
if (agent_response.text_response.empty() &&
agent_response.commands.empty() &&
agent_response.tool_calls.empty()) {
return absl::InternalError(
"❌ No valid commands extracted from Gemini response\n"
"❌ No valid response extracted from Gemini\n"
" Expected at least one of: text_response, commands, or tool_calls\n"
" Raw response: " + response_body);
}

124
src/cli/service/ai/prompt_builder.cc
View File

@@ -525,6 +525,62 @@ std::string PromptBuilder::BuildFewShotExamplesSection() const {
}
std::string PromptBuilder::BuildConstraintsSection() const {
// Try to load from file first
const std::vector<std::string> search_paths = {
"assets/agent/tool_calling_instructions.txt",
"../assets/agent/tool_calling_instructions.txt",
"../../assets/agent/tool_calling_instructions.txt",
};
for (const auto& path : search_paths) {
std::ifstream file(path);
if (file.is_open()) {
std::string content((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
if (!content.empty()) {
std::ostringstream oss;
oss << content;
// Add tool schemas if available
if (!tool_specs_.empty()) {
oss << "\n\n# Available Tools for ROM Inspection\n\n";
oss << "You have access to the following tools to answer questions:\n\n";
oss << "```json\n";
oss << BuildFunctionCallSchemas();
oss << "\n```\n\n";
oss << "**Tool Call Example (Initial Request):**\n";
oss << "```json\n";
oss << R"({
"tool_calls": [
{
"tool_name": "resource-list",
"args": {
"type": "dungeon"
}
}
],
"reasoning": "I need to call the resource-list tool to get the dungeon information."
})";
oss << "\n```\n\n";
oss << "**Tool Result Response (After Tool Executes):**\n";
oss << "```json\n";
oss << R"({
"text_response": "I found the following dungeons in the ROM: Hyrule Castle, Eastern Palace, Desert Palace, Tower of Hera, Palace of Darkness, Swamp Palace, Skull Woods, Thieves' Town, Ice Palace, Misery Mire, Turtle Rock, and Ganon's Tower.",
"reasoning": "The tool returned a list of 12 dungeons which I've formatted into a readable response."
})";
oss << "\n```\n";
}
if (!tile_reference_.empty()) {
oss << "\n" << BuildTileReferenceSection();
}
return oss.str();
}
}
}
// Fallback to embedded version if file not found
std::ostringstream oss;
oss << R"(
# Critical Constraints
@@ -541,23 +597,38 @@ std::string PromptBuilder::BuildConstraintsSection() const {
- `commands` is for generating commands to modify the ROM.
- All fields are optional, but you should always provide at least one.
2. **Tool Usage:** When the user asks a question about the ROM state, use tool_calls instead of commands
2. **Tool Calling Workflow (CRITICAL):**
WHEN YOU CALL A TOOL:
a) First response: Include tool_calls with the tool name and arguments
b) The tool will execute and you'll receive results in the next message
c) Second response: You MUST provide a text_response that answers the user's question using the tool results
d) DO NOT call the same tool again unless you need different parameters
e) DO NOT leave text_response empty after receiving tool results
Example conversation flow:
User: "What dungeons are in this ROM?"
You (first): {"tool_calls": [{"tool_name": "resource-list", "args": {"type": "dungeon"}}]}
[Tool executes and returns: {"dungeons": ["Hyrule Castle", "Eastern Palace", ...]}]
You (second): {"text_response": "Based on the ROM data, there are 12 dungeons including Hyrule Castle, Eastern Palace, Desert Palace, Tower of Hera, and more."}
3. **Tool Usage:** When the user asks a question about the ROM state, use tool_calls instead of commands
- Tools are read-only and return information
- Commands modify the ROM and should only be used when explicitly requested
- You can call multiple tools in one response
- Always use JSON format for tool results
- ALWAYS provide text_response after receiving tool results
3. **Command Syntax:** Follow the exact syntax shown in examples
4. **Command Syntax:** Follow the exact syntax shown in examples
- Use correct flag names (--group, --id, --to, --from, etc.)
- Use hex format for colors (0xRRGGBB) and tile IDs (0xNNN)
- Coordinates are 0-based indices
4. **Common Patterns:**
5. **Common Patterns:**
- Palette modifications: export → set-color → import
- Multiple tile placement: multiple overworld set-tile commands
- Validation: single rom validate command
5. **Error Prevention:**
6. **Error Prevention:**
- Always export before modifying palettes
- Use temporary file names (temp_*.json) for intermediate files
- Validate coordinates are within bounds
@@ -569,10 +640,9 @@ std::string PromptBuilder::BuildConstraintsSection() const {
oss << "```json\n";
oss << BuildFunctionCallSchemas();
oss << "\n```\n\n";
oss << "**Tool Call Example:**\n";
oss << "**Tool Call Example (Initial Request):**\n";
oss << "```json\n";
oss << R"({
"text_response": "Let me check the dungeons in this ROM.",
"tool_calls": [
{
"tool_name": "resource-list",
@@ -580,7 +650,15 @@ std::string PromptBuilder::BuildConstraintsSection() const {
"type": "dungeon"
}
}
]
],
"reasoning": "I need to call the resource-list tool to get the dungeon information."
})";
oss << "\n```\n\n";
oss << "**Tool Result Response (After Tool Executes):**\n";
oss << "```json\n";
oss << R"({
"text_response": "I found the following dungeons in the ROM: Hyrule Castle, Eastern Palace, Desert Palace, Tower of Hera, Palace of Darkness, Swamp Palace, Skull Woods, Thieves' Town, Ice Palace, Misery Mire, Turtle Rock, and Ganon's Tower.",
"reasoning": "The tool returned a list of 12 dungeons which I've formatted into a readable response."
})";
oss << "\n```\n";
}
@@ -642,6 +720,38 @@ std::string PromptBuilder::BuildContextSection(const RomContext& context) {
}
std::string PromptBuilder::BuildSystemInstruction() {
// Try to load from file first
const std::vector<std::string> search_paths = {
"assets/agent/system_prompt.txt",
"../assets/agent/system_prompt.txt",
"../../assets/agent/system_prompt.txt",
};
for (const auto& path : search_paths) {
std::ifstream file(path);
if (file.is_open()) {
std::string content((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
if (!content.empty()) {
std::ostringstream oss;
oss << content;
// Add command reference if available
if (catalogue_loaded_ && !command_docs_.empty()) {
oss << "\n\n" << BuildCommandReference();
}
// Add tool reference if available
if (!tool_specs_.empty()) {
oss << "\n\n" << BuildToolReference();
}
return oss.str();
}
}
}
// Fallback to embedded version if file not found
std::ostringstream oss;
oss << "You are an expert ROM hacking assistant for The Legend of Zelda: "

154
src/cli/util/terminal_colors.h Normal file
View File

@@ -0,0 +1,154 @@
#ifndef YAZE_SRC_CLI_UTIL_TERMINAL_COLORS_H_
#define YAZE_SRC_CLI_UTIL_TERMINAL_COLORS_H_
#include <chrono>
#include <iostream>
#include <string>
#include <thread>
namespace yaze {
namespace cli {
namespace util {
// ANSI color codes
namespace colors {
constexpr const char* kReset = "\033[0m";
constexpr const char* kBold = "\033[1m";
constexpr const char* kDim = "\033[2m";
// Regular colors
constexpr const char* kBlack = "\033[30m";
constexpr const char* kRed = "\033[31m";
constexpr const char* kGreen = "\033[32m";
constexpr const char* kYellow = "\033[33m";
constexpr const char* kBlue = "\033[34m";
constexpr const char* kMagenta = "\033[35m";
constexpr const char* kCyan = "\033[36m";
constexpr const char* kWhite = "\033[37m";
// Bright colors
constexpr const char* kBrightBlack = "\033[90m";
constexpr const char* kBrightRed = "\033[91m";
constexpr const char* kBrightGreen = "\033[92m";
constexpr const char* kBrightYellow = "\033[93m";
constexpr const char* kBrightBlue = "\033[94m";
constexpr const char* kBrightMagenta = "\033[95m";
constexpr const char* kBrightCyan = "\033[96m";
constexpr const char* kBrightWhite = "\033[97m";
// Background colors
constexpr const char* kBgBlack = "\033[40m";
constexpr const char* kBgRed = "\033[41m";
constexpr const char* kBgGreen = "\033[42m";
constexpr const char* kBgYellow = "\033[43m";
constexpr const char* kBgBlue = "\033[44m";
constexpr const char* kBgMagenta = "\033[45m";
constexpr const char* kBgCyan = "\033[46m";
constexpr const char* kBgWhite = "\033[47m";
} // namespace colors
// Icon set
namespace icons {
constexpr const char* kSuccess = "✓";
constexpr const char* kError = "✗";
constexpr const char* kWarning = "⚠";
constexpr const char* kInfo = "ℹ";
constexpr const char* kSpinner = "◐◓◑◒";
constexpr const char* kRobot = "🤖";
constexpr const char* kTool = "🔧";
constexpr const char* kThinking = "💭";
constexpr const char* kArrow = "→";
} // namespace icons
// Simple loading indicator
class LoadingIndicator {
public:
LoadingIndicator(const std::string& message, bool show = true)
: message_(message), show_(show), running_(false) {}
~LoadingIndicator() {
Stop();
}
void Start() {
if (!show_ || running_) return;
running_ = true;
thread_ = std::thread([this]() {
const char* spinner[] = {"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"};
int idx = 0;
while (running_) {
std::cout << "\r" << colors::kCyan << spinner[idx] << " "
<< colors::kBold << message_ << colors::kReset << std::flush;
idx = (idx + 1) % 10;
std::this_thread::sleep_for(std::chrono::milliseconds(80));
}
// Clear the line
std::cout << "\r" << std::string(message_.length() + 10, ' ') << "\r" << std::flush;
});
}
void Stop() {
if (running_) {
running_ = false;
if (thread_.joinable()) {
thread_.join();
}
}
}
void UpdateMessage(const std::string& message) {
message_ = message;
}
private:
std::string message_;
bool show_;
bool running_;
std::thread thread_;
};
// Utility functions for colored output
inline void PrintSuccess(const std::string& message) {
std::cout << colors::kGreen << icons::kSuccess << " " << message << colors::kReset << std::endl;
}
inline void PrintError(const std::string& message) {
std::cerr << colors::kRed << icons::kError << " " << message << colors::kReset << std::endl;
}
inline void PrintWarning(const std::string& message) {
std::cerr << colors::kYellow << icons::kWarning << " " << message << colors::kReset << std::endl;
}
inline void PrintInfo(const std::string& message) {
std::cout << colors::kBlue << icons::kInfo << " " << message << colors::kReset << std::endl;
}
inline void PrintToolCall(const std::string& tool_name, const std::string& details = "") {
std::cout << colors::kMagenta << icons::kTool << " " << colors::kBold
<< "Calling tool: " << colors::kReset << colors::kCyan << tool_name
<< colors::kReset;
if (!details.empty()) {
std::cout << colors::kDim << " (" << details << ")" << colors::kReset;
}
std::cout << std::endl;
}
inline void PrintThinking(const std::string& message = "Processing...") {
std::cout << colors::kYellow << icons::kThinking << " " << colors::kDim
<< message << colors::kReset << std::endl;
}
inline void PrintSeparator() {
std::cout << colors::kDim << "─────────────────────────────────────────"
<< colors::kReset << std::endl;
}
} // namespace util
} // namespace cli
} // namespace yaze
#endif // YAZE_SRC_CLI_UTIL_TERMINAL_COLORS_H_