feat: Add GUI Automation Tool for Tile Placement
- Introduced the HandleGuiPlaceTileCommand function to facilitate tile placement via GUI automation, enhancing the tool's capabilities for user interaction. - Updated the ToolDispatcher to include the new command, allowing for seamless integration into the existing command structure. - Enhanced the AgentChatWidget to initialize default labels for resource tools, improving the user experience with visual feedback on label loading status. - Improved error handling and user messaging for better clarity during tile placement operations.
This commit is contained in:
@@ -41,9 +41,15 @@ absl::StatusOr<std::filesystem::path> AssetLoader::FindAssetFile(const std::stri
|
||||
}
|
||||
}
|
||||
|
||||
// Debug: Print searched paths
|
||||
std::string searched_paths;
|
||||
for (const auto& path : search_paths) {
|
||||
searched_paths += "\n - " + path.string();
|
||||
}
|
||||
|
||||
return absl::NotFoundError(
|
||||
absl::StrFormat("Asset file not found: %s (searched %d paths)",
|
||||
relative_path, search_paths.size()));
|
||||
absl::StrFormat("Asset file not found: %s\nSearched paths:%s",
|
||||
relative_path, searched_paths));
|
||||
}
|
||||
|
||||
absl::StatusOr<std::string> AssetLoader::LoadTextFile(const std::string& relative_path) {
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
#include "app/editor/system/toast_manager.h"
|
||||
#include "app/gui/icons.h"
|
||||
#include "app/core/project.h"
|
||||
#include "app/rom.h"
|
||||
#include "imgui/imgui.h"
|
||||
#include "imgui/misc/cpp/imgui_stdlib.h"
|
||||
|
||||
@@ -115,6 +116,30 @@ AgentChatWidget::AgentChatWidget() {
|
||||
|
||||
void AgentChatWidget::SetRomContext(Rom* rom) {
|
||||
agent_service_.SetRomContext(rom);
|
||||
|
||||
// ALWAYS initialize embedded labels for resource tools (default Zelda3 labels)
|
||||
if (rom && rom->is_loaded() && rom->resource_label()) {
|
||||
if (!rom->resource_label()->labels_loaded_) {
|
||||
core::YazeProject project;
|
||||
|
||||
// Initialize embedded default labels (all Zelda3 resources)
|
||||
auto labels_status = project.InitializeEmbeddedLabels();
|
||||
if (labels_status.ok()) {
|
||||
rom->resource_label()->labels_ = project.resource_labels;
|
||||
rom->resource_label()->labels_loaded_ = true;
|
||||
|
||||
if (toast_manager_) {
|
||||
toast_manager_->Show(
|
||||
ICON_MD_CHECK_CIRCLE " Default Zelda3 labels loaded for AI tools",
|
||||
ToastType::kSuccess, 2.5f);
|
||||
}
|
||||
} else if (toast_manager_) {
|
||||
toast_manager_->Show(
|
||||
ICON_MD_WARNING " Warning: Could not load default labels",
|
||||
ToastType::kWarning, 3.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AgentChatWidget::SetToastManager(ToastManager* toast_manager) {
|
||||
|
||||
@@ -67,6 +67,7 @@ _yaze_ensure_yaml_cpp(YAZE_YAML_CPP_TARGET)
|
||||
set(YAZE_AGENT_SOURCES
|
||||
cli/service/agent/proposal_executor.cc
|
||||
cli/handlers/agent/tool_commands.cc
|
||||
cli/handlers/agent/gui_tool_commands.cc
|
||||
cli/handlers/agent/todo_commands.cc
|
||||
cli/handlers/agent/hex_commands.cc
|
||||
cli/handlers/agent/palette_commands.cc
|
||||
|
||||
@@ -64,6 +64,11 @@ absl::Status HandleMessageReadCommand(
|
||||
absl::Status HandleMessageSearchCommand(
|
||||
const std::vector<std::string>& arg_vec,
|
||||
Rom* rom_context = nullptr);
|
||||
|
||||
// GUI Automation Tool
|
||||
absl::Status HandleGuiPlaceTileCommand(
|
||||
const std::vector<std::string>& arg_vec,
|
||||
Rom* rom_context = nullptr);
|
||||
absl::Status HandleChatCommand(Rom& rom);
|
||||
absl::Status HandleSimpleChatCommand(const std::vector<std::string>&, Rom* rom, bool quiet);
|
||||
absl::Status HandleTestConversationCommand(
|
||||
|
||||
94
src/cli/handlers/agent/gui_tool_commands.cc
Normal file
94
src/cli/handlers/agent/gui_tool_commands.cc
Normal file
@@ -0,0 +1,94 @@
|
||||
#include "absl/strings/match.h"
|
||||
#include "cli/handlers/agent/commands.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/strings/match.h"
|
||||
#include "absl/strings/numbers.h"
|
||||
#include "app/rom.h"
|
||||
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
#include "cli/service/gui/gui_automation_client.h"
|
||||
#include "cli/service/ai/ai_action_parser.h"
|
||||
#include "cli/service/gui/gui_action_generator.h"
|
||||
#endif
|
||||
|
||||
namespace yaze {
|
||||
namespace cli {
|
||||
namespace agent {
|
||||
|
||||
absl::Status HandleGuiPlaceTileCommand(
|
||||
const std::vector<std::string>& arg_vec, Rom* rom_context) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
// Parse arguments
|
||||
int tile_id = -1;
|
||||
int x = -1;
|
||||
int y = -1;
|
||||
|
||||
for (size_t i = 0; i < arg_vec.size(); ++i) {
|
||||
const std::string& token = arg_vec[i];
|
||||
if (token == "--tile" || token == "--tile-id") {
|
||||
if (i + 1 < arg_vec.size()) {
|
||||
absl::SimpleAtoi(arg_vec[++i], &tile_id);
|
||||
}
|
||||
} else if (absl::StartsWith(token, "--tile=")) {
|
||||
absl::SimpleAtoi(token.substr(7), &tile_id);
|
||||
} else if (token == "--x") {
|
||||
if (i + 1 < arg_vec.size()) {
|
||||
absl::SimpleAtoi(arg_vec[++i], &x);
|
||||
}
|
||||
} else if (absl::StartsWith(token, "--x=")) {
|
||||
absl::SimpleAtoi(token.substr(4), &x);
|
||||
} else if (token == "--y") {
|
||||
if (i + 1 < arg_vec.size()) {
|
||||
absl::SimpleAtoi(arg_vec[++i], &y);
|
||||
}
|
||||
} else if (absl::StartsWith(token, "--y=")) {
|
||||
absl::SimpleAtoi(token.substr(4), &y);
|
||||
}
|
||||
}
|
||||
|
||||
if (tile_id < 0 || x < 0 || y < 0) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Usage: gui-place-tile --tile <id> --x <x> --y <y>");
|
||||
}
|
||||
|
||||
// Create AI actions
|
||||
ai::AIAction select_action(ai::AIActionType::kSelectTile, {});
|
||||
select_action.parameters["tile_id"] = std::to_string(tile_id);
|
||||
|
||||
ai::AIAction place_action(ai::AIActionType::kPlaceTile, {});
|
||||
place_action.parameters["x"] = std::to_string(x);
|
||||
place_action.parameters["y"] = std::to_string(y);
|
||||
|
||||
ai::AIAction save_action(ai::AIActionType::kSaveTile, {});
|
||||
|
||||
// Generate test script
|
||||
gui::GuiActionGenerator generator;
|
||||
std::vector<ai::AIAction> actions = {select_action, place_action, save_action};
|
||||
auto script_result = generator.GenerateTestScript(actions);
|
||||
if (!script_result.ok()) {
|
||||
return script_result.status();
|
||||
}
|
||||
std::string test_script = *script_result;
|
||||
|
||||
// Output as JSON for tool call response
|
||||
std::cout << "{\n";
|
||||
std::cout << " \"success\": true,\n";
|
||||
std::cout << " \"tile_id\": " << tile_id << ",\n";
|
||||
std::cout << " \"position\": {\"x\": " << x << ", \"y\": " << y << "},\n";
|
||||
std::cout << " \"test_script\": \"" << test_script << "\",\n";
|
||||
std::cout << " \"message\": \"GUI actions generated for tile placement. Use agent test execute to run.\"\n";
|
||||
std::cout << "}\n";
|
||||
|
||||
return absl::OkStatus();
|
||||
#else
|
||||
return absl::UnimplementedError("GUI automation requires YAZE_WITH_GRPC=ON");
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace agent
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
@@ -60,6 +60,9 @@ absl::StatusOr<std::string> ToolDispatcher::Dispatch(
|
||||
status = HandleMessageReadCommand(args, rom_context_);
|
||||
} else if (tool_call.tool_name == "message-search") {
|
||||
status = HandleMessageSearchCommand(args, rom_context_);
|
||||
} else if (tool_call.tool_name == "gui-place-tile") {
|
||||
// GUI automation tool for placing tiles via test harness
|
||||
status = HandleGuiPlaceTileCommand(args, rom_context_);
|
||||
} else {
|
||||
status = absl::UnimplementedError(
|
||||
absl::StrFormat("Unknown tool: %s", tool_call.tool_name));
|
||||
|
||||
Reference in New Issue
Block a user