feat: Add GUI Automation Tool for Tile Placement

- Introduced the HandleGuiPlaceTileCommand function to facilitate tile placement via GUI automation, enhancing the tool's capabilities for user interaction.
- Updated the ToolDispatcher to include the new command, allowing for seamless integration into the existing command structure.
- Enhanced the AgentChatWidget to initialize default labels for resource tools, improving the user experience with visual feedback on label loading status.
- Improved error handling and user messaging for better clarity during tile placement operations.
This commit is contained in:
scawful
2025-10-05 06:04:18 -04:00
parent 38915a5162
commit 8b9f838054
6 changed files with 136 additions and 2 deletions

View File

@@ -41,9 +41,15 @@ absl::StatusOr<std::filesystem::path> AssetLoader::FindAssetFile(const std::stri
}
}
// Debug: Print searched paths
std::string searched_paths;
for (const auto& path : search_paths) {
searched_paths += "\n - " + path.string();
}
return absl::NotFoundError(
absl::StrFormat("Asset file not found: %s (searched %d paths)",
relative_path, search_paths.size()));
absl::StrFormat("Asset file not found: %s\nSearched paths:%s",
relative_path, searched_paths));
}
absl::StatusOr<std::string> AssetLoader::LoadTextFile(const std::string& relative_path) {

View File

@@ -22,6 +22,7 @@
#include "app/editor/system/toast_manager.h"
#include "app/gui/icons.h"
#include "app/core/project.h"
#include "app/rom.h"
#include "imgui/imgui.h"
#include "imgui/misc/cpp/imgui_stdlib.h"
@@ -115,6 +116,30 @@ AgentChatWidget::AgentChatWidget() {
void AgentChatWidget::SetRomContext(Rom* rom) {
agent_service_.SetRomContext(rom);
// ALWAYS initialize embedded labels for resource tools (default Zelda3 labels)
if (rom && rom->is_loaded() && rom->resource_label()) {
if (!rom->resource_label()->labels_loaded_) {
core::YazeProject project;
// Initialize embedded default labels (all Zelda3 resources)
auto labels_status = project.InitializeEmbeddedLabels();
if (labels_status.ok()) {
rom->resource_label()->labels_ = project.resource_labels;
rom->resource_label()->labels_loaded_ = true;
if (toast_manager_) {
toast_manager_->Show(
ICON_MD_CHECK_CIRCLE " Default Zelda3 labels loaded for AI tools",
ToastType::kSuccess, 2.5f);
}
} else if (toast_manager_) {
toast_manager_->Show(
ICON_MD_WARNING " Warning: Could not load default labels",
ToastType::kWarning, 3.0f);
}
}
}
}
void AgentChatWidget::SetToastManager(ToastManager* toast_manager) {

View File

@@ -67,6 +67,7 @@ _yaze_ensure_yaml_cpp(YAZE_YAML_CPP_TARGET)
set(YAZE_AGENT_SOURCES
cli/service/agent/proposal_executor.cc
cli/handlers/agent/tool_commands.cc
cli/handlers/agent/gui_tool_commands.cc
cli/handlers/agent/todo_commands.cc
cli/handlers/agent/hex_commands.cc
cli/handlers/agent/palette_commands.cc

View File

@@ -64,6 +64,11 @@ absl::Status HandleMessageReadCommand(
absl::Status HandleMessageSearchCommand(
const std::vector<std::string>& arg_vec,
Rom* rom_context = nullptr);
// GUI Automation Tool
absl::Status HandleGuiPlaceTileCommand(
const std::vector<std::string>& arg_vec,
Rom* rom_context = nullptr);
absl::Status HandleChatCommand(Rom& rom);
absl::Status HandleSimpleChatCommand(const std::vector<std::string>&, Rom* rom, bool quiet);
absl::Status HandleTestConversationCommand(

View File

@@ -0,0 +1,94 @@
#include "absl/strings/match.h"
#include "cli/handlers/agent/commands.h"
#include <iostream>
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/strings/match.h"
#include "absl/strings/numbers.h"
#include "app/rom.h"
#ifdef YAZE_WITH_GRPC
#include "cli/service/gui/gui_automation_client.h"
#include "cli/service/ai/ai_action_parser.h"
#include "cli/service/gui/gui_action_generator.h"
#endif
namespace yaze {
namespace cli {
namespace agent {
absl::Status HandleGuiPlaceTileCommand(
const std::vector<std::string>& arg_vec, Rom* rom_context) {
#ifdef YAZE_WITH_GRPC
// Parse arguments
int tile_id = -1;
int x = -1;
int y = -1;
for (size_t i = 0; i < arg_vec.size(); ++i) {
const std::string& token = arg_vec[i];
if (token == "--tile" || token == "--tile-id") {
if (i + 1 < arg_vec.size()) {
absl::SimpleAtoi(arg_vec[++i], &tile_id);
}
} else if (absl::StartsWith(token, "--tile=")) {
absl::SimpleAtoi(token.substr(7), &tile_id);
} else if (token == "--x") {
if (i + 1 < arg_vec.size()) {
absl::SimpleAtoi(arg_vec[++i], &x);
}
} else if (absl::StartsWith(token, "--x=")) {
absl::SimpleAtoi(token.substr(4), &x);
} else if (token == "--y") {
if (i + 1 < arg_vec.size()) {
absl::SimpleAtoi(arg_vec[++i], &y);
}
} else if (absl::StartsWith(token, "--y=")) {
absl::SimpleAtoi(token.substr(4), &y);
}
}
if (tile_id < 0 || x < 0 || y < 0) {
return absl::InvalidArgumentError(
"Usage: gui-place-tile --tile <id> --x <x> --y <y>");
}
// Create AI actions
ai::AIAction select_action(ai::AIActionType::kSelectTile, {});
select_action.parameters["tile_id"] = std::to_string(tile_id);
ai::AIAction place_action(ai::AIActionType::kPlaceTile, {});
place_action.parameters["x"] = std::to_string(x);
place_action.parameters["y"] = std::to_string(y);
ai::AIAction save_action(ai::AIActionType::kSaveTile, {});
// Generate test script
gui::GuiActionGenerator generator;
std::vector<ai::AIAction> actions = {select_action, place_action, save_action};
auto script_result = generator.GenerateTestScript(actions);
if (!script_result.ok()) {
return script_result.status();
}
std::string test_script = *script_result;
// Output as JSON for tool call response
std::cout << "{\n";
std::cout << " \"success\": true,\n";
std::cout << " \"tile_id\": " << tile_id << ",\n";
std::cout << " \"position\": {\"x\": " << x << ", \"y\": " << y << "},\n";
std::cout << " \"test_script\": \"" << test_script << "\",\n";
std::cout << " \"message\": \"GUI actions generated for tile placement. Use agent test execute to run.\"\n";
std::cout << "}\n";
return absl::OkStatus();
#else
return absl::UnimplementedError("GUI automation requires YAZE_WITH_GRPC=ON");
#endif
}
} // namespace agent
} // namespace cli
} // namespace yaze

View File

@@ -60,6 +60,9 @@ absl::StatusOr<std::string> ToolDispatcher::Dispatch(
status = HandleMessageReadCommand(args, rom_context_);
} else if (tool_call.tool_name == "message-search") {
status = HandleMessageSearchCommand(args, rom_context_);
} else if (tool_call.tool_name == "gui-place-tile") {
// GUI automation tool for placing tiles via test harness
status = HandleGuiPlaceTileCommand(args, rom_context_);
} else {
status = absl::UnimplementedError(
absl::StrFormat("Unknown tool: %s", tool_call.tool_name));