From 22f0e5006ba0d1ff8c903ee28f6f5c6fda6120a5 Mon Sep 17 00:00:00 2001 From: scawful Date: Thu, 2 Oct 2025 16:56:15 -0400 Subject: [PATCH] feat: Implement widget discovery feature in GUI automation - Added `DiscoverWidgets` RPC to the ImGuiTestHarness service for enumerating GUI widgets. - Introduced `WidgetDiscoveryService` to handle widget collection and filtering based on various criteria. - Updated `agent gui discover` command to support new options for filtering and output formats. - Enhanced `GuiAutomationClient` to facilitate widget discovery requests and responses. - Added necessary protobuf messages for widget discovery in `imgui_test_harness.proto`. - Updated CLI command handling to include new GUI discovery functionality. - Improved documentation for the `agent gui discover` command with examples and output formats. --- docs/z3ed/E6-z3ed-reference.md | 66 ++-- src/app/app.cmake | 4 +- src/app/core/imgui_test_harness_service.cc | 25 ++ src/app/core/imgui_test_harness_service.h | 6 + src/app/core/proto/imgui_test_harness.proto | 60 ++++ src/app/core/widget_discovery_service.cc | 246 +++++++++++++ src/app/core/widget_discovery_service.h | 47 +++ src/cli/handlers/agent.cc | 366 +++++++++++++++++++- src/cli/modern_cli.cc | 4 +- src/cli/service/gui_automation_client.cc | 97 ++++++ src/cli/service/gui_automation_client.h | 57 +++ 11 files changed, 947 insertions(+), 31 deletions(-) create mode 100644 src/app/core/widget_discovery_service.cc create mode 100644 src/app/core/widget_discovery_service.h diff --git a/docs/z3ed/E6-z3ed-reference.md b/docs/z3ed/E6-z3ed-reference.md index fc5a1a17..2c288ff3 100644 --- a/docs/z3ed/E6-z3ed-reference.md +++ b/docs/z3ed/E6-z3ed-reference.md @@ -248,44 +248,54 @@ Examples: ##### `agent gui discover` - Enumerate available widgets ```bash -z3ed agent gui discover [--window ] [--type ] [--format ] +z3ed agent gui discover \ + [--host ] [--port ] \ + [--window ] [--path-prefix ] \ + [--type ] [--include-invisible] [--include-disabled] \ + [--format ] [--limit ] Options: - --window Filter by window name (e.g. "Overworld") - --type Filter by widget type: button, input, menu, tab, checkbox - --format Output format: json or yaml (default: yaml) + --host Harness host (default: localhost) + --port Harness port (default: 50052) + --window Filter by window name (case-insensitive substring) + --path-prefix Require widget path to start with prefix + --type Filter widget type: button, input, menu, tab, + checkbox, slider, canvas, selectable, other + --include-invisible Include widgets whose parent window is hidden + --include-disabled Include widgets flagged as disabled + --format Output as `table` (default) or `json` + --limit Maximum widgets to display (useful for large UIs) Examples: - # Discover all widgets + # Discover all widgets currently registered z3ed agent gui discover - # Find all buttons in Overworld editor + # Focus on buttons inside the Overworld editor window z3ed agent gui discover --window "Overworld" --type button - # Get JSON for AI consumption - z3ed agent gui discover --format json > widgets.json + # Export a JSON snapshot for an automation agent (showing first 50 widgets) + z3ed agent gui discover --format json --limit 50 > widgets.json ``` -**Output Example**: -```yaml -windows: - - name: Main Window - visible: true - widgets: - - id: menu_file - label: File - type: menu - enabled: true - suggested_action: "Click menuitem: File" - - name: Overworld - visible: true - widgets: - - id: btn_save - label: Save - type: button - enabled: true - position: "10,20,100,30" - suggested_action: "Click button:Save" +**Table Output Example**: +``` +=== Widget Discovery === +Server: localhost:50052 +Window filter: Overworld +Type filter: button +Include invisible: no +Include disabled: no + +Window: Overworld (visible) + • [button] Save + Path: Overworld/Toolbar/button:Save + Suggested: Click button:Save + State: visible, enabled + Bounds: (24.0, 64.0) → (112.0, 92.0) + Widget ID: 0x13fc41a2 + +Widgets shown: 3 of 18 (truncated) +Snapshot: 2025-01-16 19:42:05 ``` **Use Cases**: diff --git a/src/app/app.cmake b/src/app/app.cmake index c47fdc07..60e7d254 100644 --- a/src/app/app.cmake +++ b/src/app/app.cmake @@ -237,7 +237,9 @@ if(YAZE_WITH_GRPC) # Add service implementation sources target_sources(yaze PRIVATE ${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.cc - ${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h) + ${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h + ${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.cc + ${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.h) # Link gRPC libraries target_link_libraries(yaze PRIVATE diff --git a/src/app/core/imgui_test_harness_service.cc b/src/app/core/imgui_test_harness_service.cc index 72111afc..b49ad641 100644 --- a/src/app/core/imgui_test_harness_service.cc +++ b/src/app/core/imgui_test_harness_service.cc @@ -181,6 +181,12 @@ class ImGuiTestHarnessServiceGrpc final : public ImGuiTestHarness::Service { return ConvertStatus(impl_->GetTestResults(request, response)); } + grpc::Status DiscoverWidgets(grpc::ServerContext* context, + const DiscoverWidgetsRequest* request, + DiscoverWidgetsResponse* response) override { + return ConvertStatus(impl_->DiscoverWidgets(request, response)); + } + private: static grpc::Status ConvertStatus(const absl::Status& status) { if (status.ok()) { @@ -1146,6 +1152,25 @@ absl::Status ImGuiTestHarnessServiceImpl::GetTestResults( return absl::OkStatus(); } +absl::Status ImGuiTestHarnessServiceImpl::DiscoverWidgets( + const DiscoverWidgetsRequest* request, + DiscoverWidgetsResponse* response) { + if (!request) { + return absl::InvalidArgumentError("request cannot be null"); + } + if (!response) { + return absl::InvalidArgumentError("response cannot be null"); + } + + if (!test_manager_) { + return absl::FailedPreconditionError("TestManager not available"); + } + + widget_discovery_service_.CollectWidgets(/*ctx=*/nullptr, *request, + response); + return absl::OkStatus(); +} + // ============================================================================ // ImGuiTestHarnessServer - Server Lifecycle // ============================================================================ diff --git a/src/app/core/imgui_test_harness_service.h b/src/app/core/imgui_test_harness_service.h index b1ac78b2..6f5c2d83 100644 --- a/src/app/core/imgui_test_harness_service.h +++ b/src/app/core/imgui_test_harness_service.h @@ -8,6 +8,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "app/core/widget_discovery_service.h" // Include grpcpp headers for unique_ptr in member variable #include @@ -42,6 +43,8 @@ class ListTestsRequest; class ListTestsResponse; class GetTestResultsRequest; class GetTestResultsResponse; +class DiscoverWidgetsRequest; +class DiscoverWidgetsResponse; // Implementation of ImGuiTestHarness gRPC service // This class provides the actual RPC handlers for automated GUI testing @@ -85,9 +88,12 @@ class ImGuiTestHarnessServiceImpl { ListTestsResponse* response); absl::Status GetTestResults(const GetTestResultsRequest* request, GetTestResultsResponse* response); + absl::Status DiscoverWidgets(const DiscoverWidgetsRequest* request, + DiscoverWidgetsResponse* response); private: TestManager* test_manager_; // Non-owning pointer to access ImGuiTestEngine + WidgetDiscoveryService widget_discovery_service_; }; // Forward declaration of the gRPC service wrapper diff --git a/src/app/core/proto/imgui_test_harness.proto b/src/app/core/proto/imgui_test_harness.proto index 66bd0997..ebca3489 100644 --- a/src/app/core/proto/imgui_test_harness.proto +++ b/src/app/core/proto/imgui_test_harness.proto @@ -27,6 +27,9 @@ service ImGuiTestHarness { rpc GetTestStatus(GetTestStatusRequest) returns (GetTestStatusResponse); rpc ListTests(ListTestsRequest) returns (ListTestsResponse); rpc GetTestResults(GetTestResultsRequest) returns (GetTestResultsResponse); + + // Widget discovery API (IT-06) + rpc DiscoverWidgets(DiscoverWidgetsRequest) returns (DiscoverWidgetsResponse); } // ============================================================================ @@ -222,3 +225,60 @@ message AssertionResult { string actual_value = 4; string error_message = 5; } + +// ============================================================================ +// DiscoverWidgets - Enumerate discoverable GUI widgets +// ============================================================================ + +message DiscoverWidgetsRequest { + string window_filter = 1; // Optional: Limit to a window name (case-insensitive) + WidgetType type_filter = 2; // Optional: Limit to widget type + string path_prefix = 3; // Optional: Require widget path to start with prefix + bool include_invisible = 4; // Include widgets that are currently not visible + bool include_disabled = 5; // Include widgets that are currently disabled +} + +enum WidgetType { + WIDGET_TYPE_UNSPECIFIED = 0; + WIDGET_TYPE_ALL = 1; + WIDGET_TYPE_BUTTON = 2; + WIDGET_TYPE_INPUT = 3; + WIDGET_TYPE_MENU = 4; + WIDGET_TYPE_TAB = 5; + WIDGET_TYPE_CHECKBOX = 6; + WIDGET_TYPE_SLIDER = 7; + WIDGET_TYPE_CANVAS = 8; + WIDGET_TYPE_SELECTABLE = 9; + WIDGET_TYPE_OTHER = 10; +} + +message WidgetBounds { + float min_x = 1; + float min_y = 2; + float max_x = 3; + float max_y = 4; +} + +message DiscoveredWidget { + string path = 1; // Full hierarchical path (e.g. Overworld/Toolset/button:Pan) + string label = 2; // Human-readable label (e.g. Pan) + string type = 3; // Widget type string (button, input, ...) + string description = 4; // Description provided by registry (if any) + string suggested_action = 5; // Suggested action for automation (e.g. "Click button:Pan") + bool visible = 6; // Currently visible in UI + bool enabled = 7; // Currently enabled for interaction + WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates + uint32 widget_id = 9; // ImGui ID (debugging / direct access) +} + +message DiscoveredWindow { + string name = 1; // Window name (first segment of path) + bool visible = 2; // Whether the window is currently visible + repeated DiscoveredWidget widgets = 3; // Widgets contained in this window +} + +message DiscoverWidgetsResponse { + repeated DiscoveredWindow windows = 1; + int32 total_widgets = 2; // Total number of widgets returned + int64 generated_at_ms = 3; // Snapshot timestamp (Unix ms) +} diff --git a/src/app/core/widget_discovery_service.cc b/src/app/core/widget_discovery_service.cc new file mode 100644 index 00000000..bb936db3 --- /dev/null +++ b/src/app/core/widget_discovery_service.cc @@ -0,0 +1,246 @@ +#include "app/core/widget_discovery_service.h" + +#include +#include +#include +#include + +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace yaze { +namespace test { +namespace { + +struct WindowEntry { + int index = -1; + bool visible = true; +}; + +} // namespace + +void WidgetDiscoveryService::CollectWidgets( + ImGuiTestContext* ctx, const DiscoverWidgetsRequest& request, + DiscoverWidgetsResponse* response) const { + if (!response) { + return; + } + + response->clear_windows(); + response->set_total_widgets(0); + response->set_generated_at_ms(absl::ToUnixMillis(absl::Now())); + + const auto& registry = gui::WidgetIdRegistry::Instance().GetAllWidgets(); + if (registry.empty()) { + return; + } + + const std::string window_filter_lower = + absl::AsciiStrToLower(std::string(request.window_filter())); + const std::string path_prefix_lower = + absl::AsciiStrToLower(std::string(request.path_prefix())); + const bool include_invisible = request.include_invisible(); + const bool include_disabled = request.include_disabled(); + + std::map window_lookup; + int total_widgets = 0; + + for (const auto& [path, info] : registry) { + if (!MatchesType(info.type, request.type_filter())) { + continue; + } + + if (!MatchesPathPrefix(path, path_prefix_lower)) { + continue; + } + + const std::string window_name = ExtractWindowName(path); + if (!MatchesWindow(window_name, window_filter_lower)) { + continue; + } + + auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{}); + WindowEntry& entry = it->second; + + if (inserted) { +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE + if (ctx) { + ImGuiTestItemInfo window_info = + ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError); + entry.visible = (window_info.ID != 0); + } +#endif + } + + if (!include_invisible && !entry.visible) { + continue; + } + + if (entry.index == -1) { + DiscoveredWindow* window_proto = response->add_windows(); + entry.index = response->windows_size() - 1; + window_proto->set_name(window_name); + window_proto->set_visible(entry.visible); + } + + const std::string label = ExtractLabel(path); + + bool widget_enabled = true; + bool widget_visible = entry.visible; + +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE + bool has_item_info = false; + ImGuiTestItemInfo item_info; + if (ctx) { + item_info = ctx->ItemInfo(label.c_str(), ImGuiTestOpFlags_NoError); + if (item_info.ID != 0) { + has_item_info = true; + widget_visible = item_info.RectClipped.GetWidth() > 0.0f && + item_info.RectClipped.GetHeight() > 0.0f; + widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0; + } + } +#endif + + if (!include_invisible && !widget_visible) { + continue; + } + if (!include_disabled && !widget_enabled) { + continue; + } + + auto* window_proto = response->mutable_windows(entry.index); + auto* widget_proto = window_proto->add_widgets(); + widget_proto->set_path(path); + widget_proto->set_label(label); + widget_proto->set_type(info.type); + widget_proto->set_suggested_action(SuggestedAction(info.type, label)); + widget_proto->set_visible(widget_visible); + widget_proto->set_enabled(widget_enabled); + widget_proto->set_widget_id(info.imgui_id); + + if (!info.description.empty()) { + widget_proto->set_description(info.description); + } + + WidgetBounds* bounds = widget_proto->mutable_bounds(); +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE + if (ctx && has_item_info) { + bounds->set_min_x(item_info.RectFull.Min.x); + bounds->set_min_y(item_info.RectFull.Min.y); + bounds->set_max_x(item_info.RectFull.Max.x); + bounds->set_max_y(item_info.RectFull.Max.y); + } else { + (void)ctx; + } +#else + (void)ctx; +#endif + + ++total_widgets; + } + + response->set_total_widgets(total_widgets); +} + +bool WidgetDiscoveryService::MatchesWindow(absl::string_view window_name, + absl::string_view filter_lower) const { + if (filter_lower.empty()) { + return true; + } + std::string name_lower = absl::AsciiStrToLower(std::string(window_name)); + return absl::StrContains(name_lower, filter_lower); +} + +bool WidgetDiscoveryService::MatchesPathPrefix(absl::string_view path, + absl::string_view prefix_lower) const { + if (prefix_lower.empty()) { + return true; + } + std::string path_lower = absl::AsciiStrToLower(std::string(path)); + return absl::StartsWith(path_lower, prefix_lower); +} + +bool WidgetDiscoveryService::MatchesType(absl::string_view type, + WidgetType filter) const { + if (filter == WIDGET_TYPE_UNSPECIFIED || filter == WIDGET_TYPE_ALL) { + return true; + } + + std::string type_lower = absl::AsciiStrToLower(std::string(type)); + switch (filter) { + case WIDGET_TYPE_BUTTON: + return type_lower == "button" || type_lower == "menuitem"; + case WIDGET_TYPE_INPUT: + return type_lower == "input" || type_lower == "textbox" || + type_lower == "inputtext"; + case WIDGET_TYPE_MENU: + return type_lower == "menu" || type_lower == "menuitem"; + case WIDGET_TYPE_TAB: + return type_lower == "tab"; + case WIDGET_TYPE_CHECKBOX: + return type_lower == "checkbox"; + case WIDGET_TYPE_SLIDER: + return type_lower == "slider" || type_lower == "drag"; + case WIDGET_TYPE_CANVAS: + return type_lower == "canvas"; + case WIDGET_TYPE_SELECTABLE: + return type_lower == "selectable"; + case WIDGET_TYPE_OTHER: + return true; + default: + return true; + } +} + +std::string WidgetDiscoveryService::ExtractWindowName( + absl::string_view path) const { + size_t slash = path.find('/'); + if (slash == absl::string_view::npos) { + return std::string(path); + } + return std::string(path.substr(0, slash)); +} + +std::string WidgetDiscoveryService::ExtractLabel(absl::string_view path) const { + size_t colon = path.rfind(':'); + if (colon == absl::string_view::npos) { + size_t slash = path.rfind('/'); + if (slash == absl::string_view::npos) { + return std::string(path); + } + return std::string(path.substr(slash + 1)); + } + return std::string(path.substr(colon + 1)); +} + +std::string WidgetDiscoveryService::SuggestedAction(absl::string_view type, + absl::string_view label) const { + std::string type_lower = absl::AsciiStrToLower(std::string(type)); + if (type_lower == "button" || type_lower == "menuitem") { + return absl::StrCat("Click button:", label); + } + if (type_lower == "checkbox") { + return absl::StrCat("Toggle checkbox:", label); + } + if (type_lower == "slider" || type_lower == "drag") { + return absl::StrCat("Adjust slider:", label); + } + if (type_lower == "input" || type_lower == "textbox" || + type_lower == "inputtext") { + return absl::StrCat("Type text into:", label); + } + if (type_lower == "canvas") { + return absl::StrCat("Interact with canvas:", label); + } + if (type_lower == "tab") { + return absl::StrCat("Switch to tab:", label); + } + return absl::StrCat("Interact with:", label); +} + +} // namespace test +} // namespace yaze diff --git a/src/app/core/widget_discovery_service.h b/src/app/core/widget_discovery_service.h new file mode 100644 index 00000000..873a4a21 --- /dev/null +++ b/src/app/core/widget_discovery_service.h @@ -0,0 +1,47 @@ +#ifndef YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_ +#define YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "app/core/proto/imgui_test_harness.pb.h" +#include "app/gui/widget_id_registry.h" + +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE +#include "imgui_test_engine/imgui_te_context.h" +#else +struct ImGuiTestContext; +#endif + +namespace yaze { +namespace test { + +// Service responsible for transforming widget registry data into +// DiscoverWidgetsResponse payloads. +class WidgetDiscoveryService { + public: + WidgetDiscoveryService() = default; + + void CollectWidgets(ImGuiTestContext* ctx, + const DiscoverWidgetsRequest& request, + DiscoverWidgetsResponse* response) const; + + private: + bool MatchesWindow(absl::string_view window_name, + absl::string_view filter) const; + bool MatchesPathPrefix(absl::string_view path, + absl::string_view prefix) const; + bool MatchesType(absl::string_view type, + WidgetType filter) const; + + std::string ExtractWindowName(absl::string_view path) const; + std::string ExtractLabel(absl::string_view path) const; + std::string SuggestedAction(absl::string_view type, + absl::string_view label) const; +}; + +} // namespace test +} // namespace yaze + +#endif // YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_ diff --git a/src/cli/handlers/agent.cc b/src/cli/handlers/agent.cc index 5268ef9e..9057a4ba 100644 --- a/src/cli/handlers/agent.cc +++ b/src/cli/handlers/agent.cc @@ -25,6 +25,8 @@ #include #include +#include +#include // Declare the rom flag so we can access it ABSL_DECLARE_FLAG(std::string, rom); @@ -105,6 +107,45 @@ std::optional ParseStatusFilter(absl::string_view value) { return std::nullopt; } +std::optional ParseWidgetTypeFilter( + absl::string_view value) { + std::string lower = std::string(absl::AsciiStrToLower(value)); + if (lower.empty() || lower == "unspecified" || lower == "any") { + return WidgetTypeFilter::kUnspecified; + } + if (lower == "all") { + return WidgetTypeFilter::kAll; + } + if (lower == "button" || lower == "buttons") { + return WidgetTypeFilter::kButton; + } + if (lower == "input" || lower == "textbox" || lower == "field") { + return WidgetTypeFilter::kInput; + } + if (lower == "menu" || lower == "menuitem" || lower == "menu-item") { + return WidgetTypeFilter::kMenu; + } + if (lower == "tab" || lower == "tabs") { + return WidgetTypeFilter::kTab; + } + if (lower == "checkbox" || lower == "toggle") { + return WidgetTypeFilter::kCheckbox; + } + if (lower == "slider" || lower == "drag" || lower == "sliderfloat") { + return WidgetTypeFilter::kSlider; + } + if (lower == "canvas" || lower == "viewport") { + return WidgetTypeFilter::kCanvas; + } + if (lower == "selectable" || lower == "list-item") { + return WidgetTypeFilter::kSelectable; + } + if (lower == "other") { + return WidgetTypeFilter::kOther; + } + return std::nullopt; +} + std::string HarnessAddress(const std::string& host, int port) { return absl::StrFormat("%s:%d", host, port); } @@ -1051,6 +1092,327 @@ absl::Status HandleTestCommand(const std::vector& arg_vec) { return HandleTestRunCommand(arg_vec); } +absl::Status HandleGuiDiscoverCommand(const std::vector& arg_vec) { + std::string host = "localhost"; + int port = 50052; + std::string window_filter; + std::string path_prefix; + std::optional type_filter; + std::optional type_filter_label; + bool include_invisible = false; + bool include_disabled = false; + std::string format = "table"; + int limit = -1; + + auto require_value = [&](const std::vector& args, size_t& index, + absl::string_view flag) -> absl::StatusOr { + if (index + 1 >= args.size()) { + return absl::InvalidArgumentError( + absl::StrFormat("Flag %s requires a value", flag)); + } + return args[++index]; + }; + + for (size_t i = 0; i < arg_vec.size(); ++i) { + const std::string& token = arg_vec[i]; + if (token == "--host") { + ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--host")); + host = std::move(value); + } else if (absl::StartsWith(token, "--host=")) { + host = token.substr(7); + } else if (token == "--port") { + ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--port")); + port = std::stoi(value); + } else if (absl::StartsWith(token, "--port=")) { + port = std::stoi(token.substr(7)); + } else if (token == "--window" || token == "--window-filter") { + ASSIGN_OR_RETURN(auto value, + require_value(arg_vec, i, token.c_str())); + window_filter = std::move(value); + } else if (absl::StartsWith(token, "--window=")) { + window_filter = token.substr(9); + } else if (token == "--path-prefix") { + ASSIGN_OR_RETURN(auto value, + require_value(arg_vec, i, "--path-prefix")); + path_prefix = std::move(value); + } else if (absl::StartsWith(token, "--path-prefix=")) { + path_prefix = token.substr(14); + } else if (token == "--type") { + ASSIGN_OR_RETURN(auto value, + require_value(arg_vec, i, "--type")); + auto parsed = ParseWidgetTypeFilter(value); + if (!parsed.has_value()) { + return absl::InvalidArgumentError( + absl::StrFormat("Unknown widget type filter: %s", value)); + } + type_filter = parsed; + type_filter_label = absl::AsciiStrToLower(value); + } else if (absl::StartsWith(token, "--type=")) { + std::string value = token.substr(7); + auto parsed = ParseWidgetTypeFilter(value); + if (!parsed.has_value()) { + return absl::InvalidArgumentError( + absl::StrFormat("Unknown widget type filter: %s", value)); + } + type_filter = parsed; + type_filter_label = absl::AsciiStrToLower(value); + } else if (token == "--include-invisible") { + include_invisible = true; + } else if (token == "--include-disabled") { + include_disabled = true; + } else if (token == "--format") { + ASSIGN_OR_RETURN(auto value, + require_value(arg_vec, i, "--format")); + format = std::move(value); + } else if (absl::StartsWith(token, "--format=")) { + format = token.substr(9); + } else if (token == "--limit") { + ASSIGN_OR_RETURN(auto value, + require_value(arg_vec, i, "--limit")); + limit = std::stoi(value); + } else if (absl::StartsWith(token, "--limit=")) { + limit = std::stoi(token.substr(8)); + } else if (token == "--help" || token == "-h") { + std::cout << "Usage: agent gui discover [options]\n" + << " --host \n" + << " --port \n" + << " --window \n" + << " --type \n" + << " --path-prefix \n" + << " --include-invisible\n" + << " --include-disabled\n" + << " --format \n" + << " --limit \n"; + return absl::OkStatus(); + } else { + return absl::InvalidArgumentError( + absl::StrFormat("Unknown flag for agent gui discover: %s", + token)); + } + } + + format = absl::AsciiStrToLower(format); + if (format != "table" && format != "json") { + return absl::InvalidArgumentError( + "--format must be either 'table' or 'json'"); + } + + if (limit == 0) { + return absl::InvalidArgumentError("--limit must be positive"); + } + +#ifndef YAZE_WITH_GRPC + (void)host; + (void)port; + (void)window_filter; + (void)path_prefix; + (void)type_filter; + (void)include_invisible; + (void)include_disabled; + (void)format; + (void)limit; + return absl::UnimplementedError( + "GUI automation requires YAZE_WITH_GRPC=ON at build time.\n" + "Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON"); +#else + GuiAutomationClient client(HarnessAddress(host, port)); + RETURN_IF_ERROR(client.Connect()); + + DiscoverWidgetsQuery query; + query.window_filter = window_filter; + query.path_prefix = path_prefix; + if (type_filter.has_value()) { + query.type_filter = type_filter.value(); + } + query.include_invisible = include_invisible; + query.include_disabled = include_disabled; + + ASSIGN_OR_RETURN(auto response, client.DiscoverWidgets(query)); + + int max_items = limit > 0 ? limit : std::numeric_limits::max(); + int remaining = max_items; + std::vector trimmed_windows; + trimmed_windows.reserve(response.windows.size()); + int rendered_widgets = 0; + + for (const auto& window : response.windows) { + if (remaining <= 0) { + break; + } + DiscoveredWindowInfo trimmed; + trimmed.name = window.name; + trimmed.visible = window.visible; + + for (const auto& widget : window.widgets) { + if (remaining <= 0) { + break; + } + trimmed.widgets.push_back(widget); + --remaining; + ++rendered_widgets; + } + + if (!trimmed.widgets.empty()) { + trimmed_windows.push_back(std::move(trimmed)); + } + } + + bool truncated = rendered_widgets < response.total_widgets; + + if (format == "json") { + std::cout << "{\n"; + std::cout << " \"server\": \"" + << JsonEscape(HarnessAddress(host, port)) << "\",\n"; + std::cout << " \"totalWidgets\": " << response.total_widgets << ",\n"; + std::cout << " \"returnedWidgets\": " << rendered_widgets << ",\n"; + std::cout << " \"truncated\": " << (truncated ? "true" : "false") + << ",\n"; + std::cout << " \"generatedAt\": " + << (response.generated_at.has_value() + ? absl::StrCat("\"", + JsonEscape(absl::FormatTime( + "%Y-%m-%dT%H:%M:%SZ", + *response.generated_at, + absl::UTCTimeZone())), + "\"") + : std::string("null")) + << ",\n"; + std::cout << " \"windows\": [\n"; + + for (size_t w = 0; w < trimmed_windows.size(); ++w) { + const auto& window = trimmed_windows[w]; + std::cout << " {\n"; + std::cout << " \"name\": \"" << JsonEscape(window.name) + << "\",\n"; + std::cout << " \"visible\": " + << (window.visible ? "true" : "false") << ",\n"; + std::cout << " \"widgets\": [\n"; + for (size_t i = 0; i < window.widgets.size(); ++i) { + const auto& widget = window.widgets[i]; + std::cout << " {\n"; + std::cout << " \"path\": \"" + << JsonEscape(widget.path) << "\",\n"; + std::cout << " \"label\": \"" + << JsonEscape(widget.label) << "\",\n"; + std::cout << " \"type\": \"" + << JsonEscape(widget.type) << "\",\n"; + std::cout << " \"description\": \"" + << JsonEscape(widget.description) << "\",\n"; + std::cout << " \"suggestedAction\": \"" + << JsonEscape(widget.suggested_action) << "\",\n"; + std::cout << " \"visible\": " + << (widget.visible ? "true" : "false") << ",\n"; + std::cout << " \"enabled\": " + << (widget.enabled ? "true" : "false") << ",\n"; + std::cout << " \"bounds\": { \"min\": [" + << widget.bounds.min_x << ", " + << widget.bounds.min_y << "], \"max\": [" + << widget.bounds.max_x << ", " + << widget.bounds.max_y << "] },\n"; + std::cout << " \"widgetId\": " << widget.widget_id + << "\n"; + std::cout << " }"; + if (i + 1 < window.widgets.size()) { + std::cout << ","; + } + std::cout << "\n"; + } + std::cout << " ]\n"; + std::cout << " }"; + if (w + 1 < trimmed_windows.size()) { + std::cout << ","; + } + std::cout << "\n"; + } + + std::cout << " ]\n"; + std::cout << "}\n"; + return absl::OkStatus(); + } + + std::cout << "\n=== Widget Discovery ===\n"; + std::cout << "Server: " << HarnessAddress(host, port) << "\n"; + if (!window_filter.empty()) { + std::cout << "Window filter: " << window_filter << "\n"; + } + if (!path_prefix.empty()) { + std::cout << "Path prefix: " << path_prefix << "\n"; + } + if (type_filter_label.has_value()) { + std::cout << "Type filter: " << *type_filter_label << "\n"; + } + std::cout << "Include invisible: " << (include_invisible ? "yes" : "no") + << "\n"; + std::cout << "Include disabled: " << (include_disabled ? "yes" : "no") + << "\n\n"; + + if (trimmed_windows.empty()) { + std::cout << "No widgets matched the provided filters." << std::endl; + return absl::OkStatus(); + } + + for (const auto& window : trimmed_windows) { + std::cout << "Window: " << window.name + << (window.visible ? " (visible)" : " (hidden)") + << "\n"; + for (const auto& widget : window.widgets) { + std::cout << " • [" << widget.type << "] " << widget.label + << "\n"; + std::cout << " Path: " << widget.path << "\n"; + if (!widget.description.empty()) { + std::cout << " Description: " << widget.description + << "\n"; + } + std::cout << " Suggested: " << widget.suggested_action << "\n"; + std::cout << " State: " + << (widget.visible ? "visible" : "hidden") << ", " + << (widget.enabled ? "enabled" : "disabled") << "\n"; + std::cout << absl::StrFormat( + " Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n", + widget.bounds.min_x, widget.bounds.min_y, + widget.bounds.max_x, widget.bounds.max_y); + std::cout << " Widget ID: 0x" << std::hex << widget.widget_id + << std::dec << "\n"; + } + std::cout << "\n"; + } + + std::cout << "Widgets shown: " << rendered_widgets << " of " + << response.total_widgets; + if (truncated) { + std::cout << " (truncated)"; + } + std::cout << "\n"; + + if (response.generated_at.has_value()) { + std::cout << "Snapshot: " + << absl::FormatTime("%Y-%m-%d %H:%M:%S", + *response.generated_at, + absl::LocalTimeZone()) + << "\n"; + } + + return absl::OkStatus(); +#endif +} + +absl::Status HandleGuiCommand(const std::vector& arg_vec) { + if (arg_vec.empty()) { + return absl::InvalidArgumentError( + "Usage: agent gui [options]"); + } + + const std::string& subcommand = arg_vec[0]; + std::vector tail(arg_vec.begin() + 1, arg_vec.end()); + + if (subcommand == "discover") { + return HandleGuiDiscoverCommand(tail); + } + + return absl::InvalidArgumentError( + absl::StrFormat("Unknown agent gui subcommand: %s", subcommand)); +} + absl::Status HandleLearnCommand() { std::cout << "Agent learn not yet implemented." << std::endl; return absl::OkStatus(); @@ -1184,7 +1546,7 @@ absl::Status HandleDescribeCommand(const std::vector& arg_vec) { absl::Status Agent::Run(const std::vector& arg_vec) { if (arg_vec.empty()) { return absl::InvalidArgumentError( - "Usage: agent [options]"); + "Usage: agent [options]"); } std::string subcommand = arg_vec[0]; @@ -1198,6 +1560,8 @@ absl::Status Agent::Run(const std::vector& arg_vec) { return HandleDiffCommand(rom_, subcommand_args); } else if (subcommand == "test") { return HandleTestCommand(subcommand_args); + } else if (subcommand == "gui") { + return HandleGuiCommand(subcommand_args); } else if (subcommand == "learn") { return HandleLearnCommand(); } else if (subcommand == "list") { diff --git a/src/cli/modern_cli.cc b/src/cli/modern_cli.cc index 3ba00171..67e16372 100644 --- a/src/cli/modern_cli.cc +++ b/src/cli/modern_cli.cc @@ -60,11 +60,13 @@ void ModernCLI::SetupCommands() { commands_["agent"] = { .name = "agent", .description = "Interact with the AI agent", - .usage = "z3ed agent [options]\n" + .usage = "z3ed agent [options]\n" " test run: --prompt \"\" [--host ] [--port ] [--timeout ]\n" " test status: status --test-id [--follow] [--host ] [--port ]\n" " test list: list [--category ] [--status ] [--limit ] [--host ] [--port ]\n" " test results: results --test-id [--include-logs] [--format yaml|json] [--host ] [--port ]\n" + " gui discover: discover [--window ] [--type ] [--path-prefix ]\n" + " [--include-invisible] [--include-disabled] [--format table|json] [--limit ]\n" " describe options: [--resource ] [--format json|yaml] [--output ]\n" " [--version ] [--last-updated ]", .handler = [this](const std::vector& args) -> absl::Status { diff --git a/src/cli/service/gui_automation_client.cc b/src/cli/service/gui_automation_client.cc index 1e85c20a..39962cda 100644 --- a/src/cli/service/gui_automation_client.cc +++ b/src/cli/service/gui_automation_client.cc @@ -21,6 +21,35 @@ std::optional OptionalTimeFromMillis(int64_t millis) { return absl::FromUnixMillis(millis); } +yaze::test::WidgetType ConvertWidgetTypeFilterToProto(WidgetTypeFilter filter) { + using ProtoType = yaze::test::WidgetType; + switch (filter) { + case WidgetTypeFilter::kAll: + return ProtoType::WIDGET_TYPE_ALL; + case WidgetTypeFilter::kButton: + return ProtoType::WIDGET_TYPE_BUTTON; + case WidgetTypeFilter::kInput: + return ProtoType::WIDGET_TYPE_INPUT; + case WidgetTypeFilter::kMenu: + return ProtoType::WIDGET_TYPE_MENU; + case WidgetTypeFilter::kTab: + return ProtoType::WIDGET_TYPE_TAB; + case WidgetTypeFilter::kCheckbox: + return ProtoType::WIDGET_TYPE_CHECKBOX; + case WidgetTypeFilter::kSlider: + return ProtoType::WIDGET_TYPE_SLIDER; + case WidgetTypeFilter::kCanvas: + return ProtoType::WIDGET_TYPE_CANVAS; + case WidgetTypeFilter::kSelectable: + return ProtoType::WIDGET_TYPE_SELECTABLE; + case WidgetTypeFilter::kOther: + return ProtoType::WIDGET_TYPE_OTHER; + case WidgetTypeFilter::kUnspecified: + default: + return ProtoType::WIDGET_TYPE_UNSPECIFIED; + } +} + TestRunStatus ConvertStatusProto( yaze::test::GetTestStatusResponse::Status status) { using ProtoStatus = yaze::test::GetTestStatusResponse::Status; @@ -440,5 +469,73 @@ absl::StatusOr GuiAutomationClient::GetTestResults( #endif } +absl::StatusOr GuiAutomationClient::DiscoverWidgets( + const DiscoverWidgetsQuery& query) { +#ifdef YAZE_WITH_GRPC + if (!stub_) { + return absl::FailedPreconditionError("Not connected. Call Connect() first."); + } + + yaze::test::DiscoverWidgetsRequest request; + if (!query.window_filter.empty()) { + request.set_window_filter(query.window_filter); + } + request.set_type_filter(ConvertWidgetTypeFilterToProto(query.type_filter)); + if (!query.path_prefix.empty()) { + request.set_path_prefix(query.path_prefix); + } + request.set_include_invisible(query.include_invisible); + request.set_include_disabled(query.include_disabled); + + yaze::test::DiscoverWidgetsResponse response; + grpc::ClientContext context; + grpc::Status status = stub_->DiscoverWidgets(&context, request, &response); + + if (!status.ok()) { + return absl::InternalError( + absl::StrFormat("DiscoverWidgets RPC failed: %s", + status.error_message())); + } + + DiscoverWidgetsResult result; + result.total_widgets = response.total_widgets(); + if (response.generated_at_ms() > 0) { + result.generated_at = OptionalTimeFromMillis(response.generated_at_ms()); + } + + result.windows.reserve(response.windows_size()); + for (const auto& window_proto : response.windows()) { + DiscoveredWindowInfo window_info; + window_info.name = window_proto.name(); + window_info.visible = window_proto.visible(); + window_info.widgets.reserve(window_proto.widgets_size()); + + for (const auto& widget_proto : window_proto.widgets()) { + WidgetDescriptor widget; + widget.path = widget_proto.path(); + widget.label = widget_proto.label(); + widget.type = widget_proto.type(); + widget.description = widget_proto.description(); + widget.suggested_action = widget_proto.suggested_action(); + widget.visible = widget_proto.visible(); + widget.enabled = widget_proto.enabled(); + widget.bounds.min_x = widget_proto.bounds().min_x(); + widget.bounds.min_y = widget_proto.bounds().min_y(); + widget.bounds.max_x = widget_proto.bounds().max_x(); + widget.bounds.max_y = widget_proto.bounds().max_y(); + widget.widget_id = widget_proto.widget_id(); + window_info.widgets.push_back(std::move(widget)); + } + + result.windows.push_back(std::move(window_info)); + } + + return result; +#else + (void)query; + return absl::UnimplementedError("gRPC not available"); +#endif +} + } // namespace cli } // namespace yaze diff --git a/src/cli/service/gui_automation_client.h b/src/cli/service/gui_automation_client.h index a613cd06..4f3ae678 100644 --- a/src/cli/service/gui_automation_client.h +++ b/src/cli/service/gui_automation_client.h @@ -9,6 +9,7 @@ #include "absl/time/time.h" #include +#include #include #include #include @@ -120,6 +121,59 @@ struct TestResultDetails { std::map metrics; }; +enum class WidgetTypeFilter { + kUnspecified, + kAll, + kButton, + kInput, + kMenu, + kTab, + kCheckbox, + kSlider, + kCanvas, + kSelectable, + kOther, +}; + +struct WidgetBoundingBox { + float min_x = 0.0f; + float min_y = 0.0f; + float max_x = 0.0f; + float max_y = 0.0f; +}; + +struct WidgetDescriptor { + std::string path; + std::string label; + std::string type; + std::string description; + std::string suggested_action; + bool visible = true; + bool enabled = true; + WidgetBoundingBox bounds; + uint32_t widget_id = 0; +}; + +struct DiscoveredWindowInfo { + std::string name; + bool visible = true; + std::vector widgets; +}; + +struct DiscoverWidgetsQuery { + std::string window_filter; + WidgetTypeFilter type_filter = WidgetTypeFilter::kUnspecified; + std::string path_prefix; + bool include_invisible = false; + bool include_disabled = false; +}; + +struct DiscoverWidgetsResult { + std::vector windows; + int total_widgets = 0; + std::optional generated_at; +}; + /** * @brief Client for automating YAZE GUI through gRPC * @@ -225,6 +279,9 @@ class GuiAutomationClient { absl::StatusOr GetTestResults(const std::string& test_id, bool include_logs = false); + absl::StatusOr DiscoverWidgets( + const DiscoverWidgetsQuery& query); + /** * @brief Check if client is connected */