Enhance AI Agent Integration and Tool Command Functionality

- Added support for JSON in CMake configuration for AI integrations.
- Implemented new tool commands: resource-list and dungeon-list-sprites.
- Created ToolDispatcher for managing tool command execution.
- Refactored CMake structure to include agent sources and improve build configuration.
- Updated agent roadmap and README documentation to reflect current status and next steps.
This commit is contained in:
scawful
2025-10-03 13:34:54 -04:00
parent 7c2bf8e1c7
commit ba9f6533a4
12 changed files with 338 additions and 225 deletions

46
src/cli/agent.cmake Normal file
View File

@@ -0,0 +1,46 @@
set(YAZE_AGENT_SOURCES
cli/handlers/agent/tool_commands.cc
cli/service/agent/conversational_agent_service.cc
cli/service/agent/tool_dispatcher.cc
cli/service/ai/ai_service.cc
cli/service/ai/ollama_ai_service.cc
cli/service/ai/prompt_builder.cc
cli/service/ai/service_factory.cc
cli/service/planning/policy_evaluator.cc
cli/service/planning/proposal_registry.cc
cli/service/planning/tile16_proposal_generator.cc
cli/service/resources/resource_catalog.cc
cli/service/resources/resource_context_builder.cc
cli/service/rom/rom_sandbox_manager.cc
)
if(YAZE_WITH_JSON)
list(APPEND YAZE_AGENT_SOURCES cli/service/ai/gemini_ai_service.cc)
endif()
add_library(yaze_agent STATIC ${YAZE_AGENT_SOURCES})
target_link_libraries(yaze_agent
PUBLIC
yaze_common
${ABSL_TARGETS}
)
target_include_directories(yaze_agent
PUBLIC
${CMAKE_SOURCE_DIR}/src
${CMAKE_SOURCE_DIR}/incl
${CMAKE_SOURCE_DIR}/third_party/httplib
${CMAKE_SOURCE_DIR}/src/lib
)
if(SDL2_INCLUDE_DIR)
target_include_directories(yaze_agent PUBLIC ${SDL2_INCLUDE_DIR})
endif()
if(YAZE_WITH_JSON)
target_link_libraries(yaze_agent PUBLIC nlohmann_json::nlohmann_json)
target_compile_definitions(yaze_agent PUBLIC YAZE_WITH_JSON)
endif()
set_target_properties(yaze_agent PROPERTIES POSITION_INDEPENDENT_CODE ON)

View File

@@ -470,160 +470,6 @@ absl::Status HandleDescribeCommand(const std::vector<std::string>& arg_vec) {
return absl::OkStatus();
}
absl::Status HandleResourceListCommand(
const std::vector<std::string>& arg_vec) {
std::string type;
std::string format = "table";
for (size_t i = 0; i < arg_vec.size(); ++i) {
const std::string& token = arg_vec[i];
if (token == "--type") {
if (i + 1 < arg_vec.size()) {
type = arg_vec[++i];
} else {
return absl::InvalidArgumentError("--type requires a value.");
}
} else if (absl::StartsWith(token, "--type=")) {
type = token.substr(7);
} else if (token == "--format") {
if (i + 1 < arg_vec.size()) {
format = arg_vec[++i];
} else {
return absl::InvalidArgumentError("--format requires a value.");
}
} else if (absl::StartsWith(token, "--format=")) {
format = token.substr(9);
}
}
if (type.empty()) {
return absl::InvalidArgumentError(
"Usage: agent resource-list --type <type> [--format <table|json>]");
}
// 1. Load the ROM
std::string rom_path = absl::GetFlag(FLAGS_rom);
if (rom_path.empty()) {
return absl::FailedPreconditionError(
"No ROM loaded. Use --rom=<path> to specify ROM file.");
}
Rom rom;
auto status = rom.LoadFromFile(rom_path);
if (!status.ok()) {
return absl::FailedPreconditionError(absl::StrFormat(
"Failed to load ROM from '%s': %s", rom_path, status.message()));
}
// 2. Get labels using ResourceContextBuilder
ResourceContextBuilder context_builder(&rom);
auto labels_or = context_builder.GetLabels(type);
if (!labels_or.ok()) {
return labels_or.status();
}
auto labels = labels_or.value();
// 3. Format and print output
if (format == "json") {
std::cout << "{\n";
bool first = true;
for (const auto& [key, value] : labels) {
if (!first) {
std::cout << ",\n";
}
std::cout << " \"" << key << "\": \"" << value << "\"";
first = false;
}
std::cout << "\n}\n";
} else { // Table format
std::cout << "=== " << absl::AsciiStrToUpper(type) << " Labels ===\n";
for (const auto& [key, value] : labels) {
std::cout << absl::StrFormat(" %-10s : %s\n", key, value);
}
}
return absl::OkStatus();
}
absl::Status HandleDungeonListSpritesCommand(
const std::vector<std::string>& arg_vec) {
std::string room_id_str;
std::string format = "table";
for (size_t i = 0; i < arg_vec.size(); ++i) {
const std::string& token = arg_vec[i];
if (token == "--room") {
if (i + 1 < arg_vec.size()) {
room_id_str = arg_vec[++i];
} else {
return absl::InvalidArgumentError("--room requires a value.");
}
} else if (absl::StartsWith(token, "--room=")) {
room_id_str = token.substr(7);
} else if (token == "--format") {
if (i + 1 < arg_vec.size()) {
format = arg_vec[++i];
} else {
return absl::InvalidArgumentError("--format requires a value.");
}
} else if (absl::StartsWith(token, "--format=")) {
format = token.substr(9);
}
}
if (room_id_str.empty()) {
return absl::InvalidArgumentError(
"Usage: agent dungeon-list-sprites --room <id> [--format "
"<table|json>]");
}
int room_id;
if (!absl::SimpleHexAtoi(room_id_str, &room_id)) {
return absl::InvalidArgumentError("Invalid room ID format. Must be hex.");
}
// 1. Load the ROM
std::string rom_path = absl::GetFlag(FLAGS_rom);
if (rom_path.empty()) {
return absl::FailedPreconditionError(
"No ROM loaded. Use --rom=<path> to specify ROM file.");
}
Rom rom;
auto status = rom.LoadFromFile(rom_path);
if (!status.ok()) {
return absl::FailedPreconditionError(absl::StrFormat(
"Failed to load ROM from '%s': %s", rom_path, status.message()));
}
// 2. Load dungeon room and get sprites
auto room = zelda3::LoadRoomFromRom(&rom, room_id);
const auto& sprites = room.GetSprites();
// 3. Format and print output
if (format == "json") {
std::cout << "[\n";
for (size_t i = 0; i < sprites.size(); ++i) {
const auto& sprite = sprites[i];
std::cout << " {\n";
std::cout << " \"id\": " << sprite.id() << ",\n";
std::cout << " \"x\": " << sprite.x() << ",\n";
std::cout << " \"y\": " << sprite.y() << "\n";
std::cout << " }" << (i == sprites.size() - 1 ? "" : ",");
std::cout << "\n";
}
std::cout << "]\n";
} else { // Table format
std::cout << "=== Sprites in Room " << room_id_str << " ===\n";
std::cout << absl::StrFormat("%-10s %-5s %-5s\n", "ID (Hex)", "X", "Y");
std::cout << std::string(22, '-') << "\n";
for (const auto& sprite : sprites) {
std::cout << absl::StrFormat("0x%-8X %-5d %-5d\n", sprite.id(),
sprite.x(), sprite.y());
}
}
return absl::OkStatus();
}
absl::Status HandleChatCommand() {
tui::ChatTUI chat_tui;
chat_tui.Run();

View File

@@ -0,0 +1,180 @@
#include "cli/handlers/agent/commands.h"
#include <iostream>
#include <string>
#include <utility>
#include "absl/flags/declare.h"
#include "absl/flags/flag.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/numbers.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "app/rom.h"
#include "app/zelda3/dungeon/room.h"
#include "cli/service/resources/resource_context_builder.h"
ABSL_DECLARE_FLAG(std::string, rom);
namespace yaze {
namespace cli {
namespace agent {
namespace {
absl::StatusOr<Rom> LoadRomFromFlag() {
std::string rom_path = absl::GetFlag(FLAGS_rom);
if (rom_path.empty()) {
return absl::FailedPreconditionError(
"No ROM loaded. Use --rom=<path> to specify ROM file.");
}
Rom rom;
auto status = rom.LoadFromFile(rom_path);
if (!status.ok()) {
return absl::FailedPreconditionError(absl::StrFormat(
"Failed to load ROM from '%s': %s", rom_path, status.message()));
}
return rom;
}
} // namespace
absl::Status HandleResourceListCommand(
const std::vector<std::string>& arg_vec) {
std::string type;
std::string format = "table";
for (size_t i = 0; i < arg_vec.size(); ++i) {
const std::string& token = arg_vec[i];
if (token == "--type") {
if (i + 1 >= arg_vec.size()) {
return absl::InvalidArgumentError("--type requires a value.");
}
type = arg_vec[++i];
} else if (absl::StartsWith(token, "--type=")) {
type = token.substr(7);
} else if (token == "--format") {
if (i + 1 >= arg_vec.size()) {
return absl::InvalidArgumentError("--format requires a value.");
}
format = arg_vec[++i];
} else if (absl::StartsWith(token, "--format=")) {
format = token.substr(9);
}
}
if (type.empty()) {
return absl::InvalidArgumentError(
"Usage: agent resource-list --type <type> [--format <table|json>]");
}
auto rom_or = LoadRomFromFlag();
if (!rom_or.ok()) {
return rom_or.status();
}
Rom rom = std::move(rom_or.value());
ResourceContextBuilder context_builder(&rom);
auto labels_or = context_builder.GetLabels(type);
if (!labels_or.ok()) {
return labels_or.status();
}
auto labels = std::move(labels_or.value());
if (format == "json") {
std::cout << "{\n";
bool first = true;
for (const auto& [key, value] : labels) {
if (!first) {
std::cout << ",\n";
}
std::cout << " \"" << key << "\": \"" << value << "\"";
first = false;
}
std::cout << "\n}\n";
} else {
std::cout << "=== " << absl::AsciiStrToUpper(type) << " Labels ===\n";
for (const auto& [key, value] : labels) {
std::cout << absl::StrFormat(" %-10s : %s\n", key, value);
}
}
return absl::OkStatus();
}
absl::Status HandleDungeonListSpritesCommand(
const std::vector<std::string>& arg_vec) {
std::string room_id_str;
std::string format = "table";
for (size_t i = 0; i < arg_vec.size(); ++i) {
const std::string& token = arg_vec[i];
if (token == "--room") {
if (i + 1 >= arg_vec.size()) {
return absl::InvalidArgumentError("--room requires a value.");
}
room_id_str = arg_vec[++i];
} else if (absl::StartsWith(token, "--room=")) {
room_id_str = token.substr(7);
} else if (token == "--format") {
if (i + 1 >= arg_vec.size()) {
return absl::InvalidArgumentError("--format requires a value.");
}
format = arg_vec[++i];
} else if (absl::StartsWith(token, "--format=")) {
format = token.substr(9);
}
}
if (room_id_str.empty()) {
return absl::InvalidArgumentError(
"Usage: agent dungeon-list-sprites --room <id> [--format <table|json>]");
}
int room_id;
if (!absl::SimpleHexAtoi(room_id_str, &room_id)) {
return absl::InvalidArgumentError(
"Invalid room ID format. Must be hex.");
}
auto rom_or = LoadRomFromFlag();
if (!rom_or.ok()) {
return rom_or.status();
}
Rom rom = std::move(rom_or.value());
auto room = zelda3::LoadRoomFromRom(&rom, room_id);
const auto& sprites = room.GetSprites();
if (format == "json") {
std::cout << "[\n";
for (size_t i = 0; i < sprites.size(); ++i) {
const auto& sprite = sprites[i];
std::cout << " {\n";
std::cout << " \"id\": " << sprite.id() << ",\n";
std::cout << " \"x\": " << sprite.x() << ",\n";
std::cout << " \"y\": " << sprite.y() << "\n";
std::cout << " }" << (i + 1 == sprites.size() ? "" : ",") << "\n";
}
std::cout << "]\n";
} else {
std::cout << "=== Sprites in Room " << room_id_str << " ===\n";
std::cout << absl::StrFormat("%-10s %-5s %-5s\n", "ID (Hex)", "X", "Y");
std::cout << std::string(22, '-') << "\n";
for (const auto& sprite : sprites) {
std::cout << absl::StrFormat("0x%-8X %-5d %-5d\n", sprite.id(),
sprite.x(), sprite.y());
}
}
return absl::OkStatus();
}
} // namespace agent
} // namespace cli
} // namespace yaze

View File

@@ -1,5 +1,8 @@
#include "cli/service/agent/tool_dispatcher.h"
#include <iostream>
#include <sstream>
#include "absl/strings/str_format.h"
#include "cli/handlers/agent/commands.h"
@@ -15,24 +18,29 @@ absl::StatusOr<std::string> ToolDispatcher::Dispatch(
args.push_back(value);
}
// Capture stdout
std::stringstream buffer;
auto old_cout_buf = std::cout.rdbuf();
std::cout.rdbuf(buffer.rdbuf());
absl::Status status;
if (tool_call.tool_name == "resource-list") {
// Note: This is a simplified approach for now. A more robust solution
// would capture stdout instead of relying on the handler to return a string.
auto status = HandleResourceListCommand(args);
if (!status.ok()) {
return status;
}
return "Successfully listed resources.";
status = HandleResourceListCommand(args);
} else if (tool_call.tool_name == "dungeon-list-sprites") {
auto status = HandleDungeonListSpritesCommand(args);
if (!status.ok()) {
return status;
}
return "Successfully listed sprites.";
status = HandleDungeonListSpritesCommand(args);
} else {
status = absl::UnimplementedError(
absl::StrFormat("Unknown tool: %s", tool_call.tool_name));
}
return absl::UnimplementedError(
absl::StrFormat("Unknown tool: %s", tool_call.tool_name));
// Restore stdout
std::cout.rdbuf(old_cout_buf);
if (!status.ok()) {
return status;
}
return buffer.str();
}
} // namespace agent

View File

@@ -1,23 +1,32 @@
#include "cli/service/ai/service_factory.h"
#include <cstring>
#include <iostream>
#include "cli/service/ai/ai_service.h"
#include "cli/service/ai/gemini_ai_service.h"
#include "cli/service/ai/ollama_ai_service.h"
#ifdef YAZE_WITH_JSON
#include "cli/service/ai/gemini_ai_service.h"
#endif
namespace yaze {
namespace cli {
std::unique_ptr<AIService> CreateAIService() {
// Priority: Ollama (local) > Gemini (remote) > Mock (testing)
const char* provider_env = std::getenv("YAZE_AI_PROVIDER");
const char* gemini_key = std::getenv("GEMINI_API_KEY");
const char* ollama_model = std::getenv("OLLAMA_MODEL");
const std::string provider = provider_env ? provider_env : "";
const bool gemini_requested = provider == "gemini";
#ifdef YAZE_WITH_JSON
const char* gemini_key = std::getenv("GEMINI_API_KEY");
const char* gemini_model = std::getenv("GEMINI_MODEL");
#endif
// Explicit provider selection
if (provider_env && std::string(provider_env) == "ollama") {
if (provider == "ollama") {
OllamaConfig config;
// Allow model override via env
@@ -39,6 +48,7 @@ std::unique_ptr<AIService> CreateAIService() {
}
// Gemini if API key provided
#ifdef YAZE_WITH_JSON
if (gemini_key && std::strlen(gemini_key) > 0) {
GeminiConfig config(gemini_key);
@@ -59,6 +69,11 @@ std::unique_ptr<AIService> CreateAIService() {
std::cout << "🤖 Using Gemini AI with model: " << config.model << std::endl;
return service;
}
#else
if (gemini_requested || std::getenv("GEMINI_API_KEY")) {
std::cerr << "⚠️ Gemini support not available: rebuild with YAZE_WITH_JSON=ON" << std::endl;
}
#endif
// Default: Mock service for testing
std::cout << "🤖 Using MockAIService (no LLM configured)" << std::endl;

View File

@@ -48,13 +48,6 @@ add_executable(
cli/handlers/agent/test_common.cc
cli/handlers/agent/test_commands.cc
cli/handlers/agent/gui_commands.cc
cli/service/ai/ai_service.cc
cli/service/ai/ollama_ai_service.cc
cli/service/ai/prompt_builder.cc
cli/service/planning/proposal_registry.cc
cli/service/resources/resource_catalog.cc
cli/service/rom/rom_sandbox_manager.cc
cli/service/planning/policy_evaluator.cc
cli/service/testing/test_suite.h
cli/service/testing/test_suite_loader.cc
cli/service/testing/test_suite_loader.h
@@ -62,17 +55,11 @@ add_executable(
cli/service/testing/test_suite_reporter.h
cli/service/testing/test_suite_writer.cc
cli/service/testing/test_suite_writer.h
cli/service/ai/gemini_ai_service.cc
cli/service/planning/tile16_proposal_generator.h
cli/service/planning/tile16_proposal_generator.cc
cli/service/resources/resource_context_builder.h
cli/service/resources/resource_context_builder.cc
cli/service/agent/conversational_agent_service.h
cli/service/agent/conversational_agent_service.cc
cli/service/ai/service_factory.h
cli/service/ai/service_factory.cc
cli/service/agent/tool_dispatcher.h
cli/service/agent/tool_dispatcher.cc
app/rom.cc
app/core/project.cc
app/core/asar_wrapper.cc
@@ -85,13 +72,8 @@ add_executable(
${IMGUI_SRC}
)
option(YAZE_WITH_JSON "Build with JSON support" OFF)
if(YAZE_WITH_JSON)
add_subdirectory(${CMAKE_SOURCE_DIR}/third_party/json ${CMAKE_BINARY_DIR}/third_party/json)
target_compile_definitions(z3ed PRIVATE YAZE_WITH_JSON)
target_link_libraries(z3ed PRIVATE nlohmann_json::nlohmann_json)
list(APPEND Z3ED_SRC_FILES cli/service/ai/gemini_ai_service.cc)
list(APPEND Z3ED_SRC_FILES cli/service/ai/prompt_builder.cc)
endif()
# ============================================================================
@@ -144,6 +126,7 @@ target_include_directories(
target_link_libraries(
z3ed PUBLIC
asar-static
yaze_agent
ftxui::component
ftxui::screen
ftxui::dom