scawful/yaze
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement widget discovery feature in GUI automation

Browse Source 
- Added `DiscoverWidgets` RPC to the ImGuiTestHarness service for enumerating GUI widgets.
- Introduced `WidgetDiscoveryService` to handle widget collection and filtering based on various criteria.
- Updated `agent gui discover` command to support new options for filtering and output formats.
- Enhanced `GuiAutomationClient` to facilitate widget discovery requests and responses.
- Added necessary protobuf messages for widget discovery in `imgui_test_harness.proto`.
- Updated CLI command handling to include new GUI discovery functionality.
- Improved documentation for the `agent gui discover` command with examples and output formats.
This commit is contained in:
scawful
2025-10-02 16:56:15 -04:00
parent 3944861b38
commit 22f0e5006b
11 changed files with 947 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
docs/z3ed/E6-z3ed-reference.md
View File

@@ -248,44 +248,54 @@ Examples:
##### `agent gui discover` - Enumerate available widgets
```bash
z3ed agent gui discover [--window <name>] [--type <widget_type>] [--format <json|yaml>]
z3ed agent gui discover \
[--host <name>] [--port <port>] \
[--window <name>] [--path-prefix <path>] \
[--type <widget_type>] [--include-invisible] [--include-disabled] \
[--format <table|json>] [--limit <n>]
Options:
--window <name> Filter by window name (e.g. "Overworld")
--type <type> Filter by widget type: button, input, menu, tab, checkbox
--format <format> Output format: json or yaml (default: yaml)
--host <name> Harness host (default: localhost)
--port <port> Harness port (default: 50052)
--window <name> Filter by window name (case-insensitive substring)
--path-prefix <path> Require widget path to start with prefix
--type <type> Filter widget type: button, input, menu, tab,
checkbox, slider, canvas, selectable, other
--include-invisible Include widgets whose parent window is hidden
--include-disabled Include widgets flagged as disabled
--format <mode> Output as `table` (default) or `json`
--limit <n> Maximum widgets to display (useful for large UIs)
Examples:
# Discover all widgets
# Discover all widgets currently registered
z3ed agent gui discover
# Find all buttons in Overworld editor
# Focus on buttons inside the Overworld editor window
z3ed agent gui discover --window "Overworld" --type button
# Get JSON for AI consumption
z3ed agent gui discover --format json > widgets.json
# Export a JSON snapshot for an automation agent (showing first 50 widgets)
z3ed agent gui discover --format json --limit 50 > widgets.json
```
**Output Example**:
```yaml
windows:
- name: Main Window
visible: true
widgets:
- id: menu_file
label: File
type: menu
enabled: true
suggested_action: "Click menuitem: File"
- name: Overworld
visible: true
widgets:
- id: btn_save
label: Save
type: button
enabled: true
position: "10,20,100,30"
suggested_action: "Click button:Save"
**Table Output Example**:
```
=== Widget Discovery ===
Server: localhost:50052
Window filter: Overworld
Type filter: button
Include invisible: no
Include disabled: no
Window: Overworld (visible)
• [button] Save
Path: Overworld/Toolbar/button:Save
Suggested: Click button:Save
State: visible, enabled
Bounds: (24.0, 64.0) → (112.0, 92.0)
Widget ID: 0x13fc41a2
Widgets shown: 3 of 18 (truncated)
Snapshot: 2025-01-16 19:42:05
```
**Use Cases**:

4
src/app/app.cmake
View File

@@ -237,7 +237,9 @@ if(YAZE_WITH_GRPC)
# Add service implementation sources
target_sources(yaze PRIVATE
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.cc
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h)
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h
${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.cc
${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.h)
# Link gRPC libraries
target_link_libraries(yaze PRIVATE

25
src/app/core/imgui_test_harness_service.cc
View File

@@ -181,6 +181,12 @@ class ImGuiTestHarnessServiceGrpc final : public ImGuiTestHarness::Service {
return ConvertStatus(impl_->GetTestResults(request, response));
}
grpc::Status DiscoverWidgets(grpc::ServerContext* context,
const DiscoverWidgetsRequest* request,
DiscoverWidgetsResponse* response) override {
return ConvertStatus(impl_->DiscoverWidgets(request, response));
}
private:
static grpc::Status ConvertStatus(const absl::Status& status) {
if (status.ok()) {
@@ -1146,6 +1152,25 @@ absl::Status ImGuiTestHarnessServiceImpl::GetTestResults(
return absl::OkStatus();
}
absl::Status ImGuiTestHarnessServiceImpl::DiscoverWidgets(
const DiscoverWidgetsRequest* request,
DiscoverWidgetsResponse* response) {
if (!request) {
return absl::InvalidArgumentError("request cannot be null");
}
if (!response) {
return absl::InvalidArgumentError("response cannot be null");
}
if (!test_manager_) {
return absl::FailedPreconditionError("TestManager not available");
}
widget_discovery_service_.CollectWidgets(/*ctx=*/nullptr, *request,
response);
return absl::OkStatus();
}
// ============================================================================
// ImGuiTestHarnessServer - Server Lifecycle
// ============================================================================

6
src/app/core/imgui_test_harness_service.h
View File

@@ -8,6 +8,7 @@
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "app/core/widget_discovery_service.h"
// Include grpcpp headers for unique_ptr<Server> in member variable
#include <grpcpp/server.h>
@@ -42,6 +43,8 @@ class ListTestsRequest;
class ListTestsResponse;
class GetTestResultsRequest;
class GetTestResultsResponse;
class DiscoverWidgetsRequest;
class DiscoverWidgetsResponse;
// Implementation of ImGuiTestHarness gRPC service
// This class provides the actual RPC handlers for automated GUI testing
@@ -85,9 +88,12 @@ class ImGuiTestHarnessServiceImpl {
ListTestsResponse* response);
absl::Status GetTestResults(const GetTestResultsRequest* request,
GetTestResultsResponse* response);
absl::Status DiscoverWidgets(const DiscoverWidgetsRequest* request,
DiscoverWidgetsResponse* response);
private:
TestManager* test_manager_; // Non-owning pointer to access ImGuiTestEngine
WidgetDiscoveryService widget_discovery_service_;
};
// Forward declaration of the gRPC service wrapper

60
src/app/core/proto/imgui_test_harness.proto
View File

@@ -27,6 +27,9 @@ service ImGuiTestHarness {
rpc GetTestStatus(GetTestStatusRequest) returns (GetTestStatusResponse);
rpc ListTests(ListTestsRequest) returns (ListTestsResponse);
rpc GetTestResults(GetTestResultsRequest) returns (GetTestResultsResponse);
// Widget discovery API (IT-06)
rpc DiscoverWidgets(DiscoverWidgetsRequest) returns (DiscoverWidgetsResponse);
}
// ============================================================================
@@ -222,3 +225,60 @@ message AssertionResult {
string actual_value = 4;
string error_message = 5;
}
// ============================================================================
// DiscoverWidgets - Enumerate discoverable GUI widgets
// ============================================================================
message DiscoverWidgetsRequest {
string window_filter = 1; // Optional: Limit to a window name (case-insensitive)
WidgetType type_filter = 2; // Optional: Limit to widget type
string path_prefix = 3; // Optional: Require widget path to start with prefix
bool include_invisible = 4; // Include widgets that are currently not visible
bool include_disabled = 5; // Include widgets that are currently disabled
}
enum WidgetType {
WIDGET_TYPE_UNSPECIFIED = 0;
WIDGET_TYPE_ALL = 1;
WIDGET_TYPE_BUTTON = 2;
WIDGET_TYPE_INPUT = 3;
WIDGET_TYPE_MENU = 4;
WIDGET_TYPE_TAB = 5;
WIDGET_TYPE_CHECKBOX = 6;
WIDGET_TYPE_SLIDER = 7;
WIDGET_TYPE_CANVAS = 8;
WIDGET_TYPE_SELECTABLE = 9;
WIDGET_TYPE_OTHER = 10;
}
message WidgetBounds {
float min_x = 1;
float min_y = 2;
float max_x = 3;
float max_y = 4;
}
message DiscoveredWidget {
string path = 1; // Full hierarchical path (e.g. Overworld/Toolset/button:Pan)
string label = 2; // Human-readable label (e.g. Pan)
string type = 3; // Widget type string (button, input, ...)
string description = 4; // Description provided by registry (if any)
string suggested_action = 5; // Suggested action for automation (e.g. "Click button:Pan")
bool visible = 6; // Currently visible in UI
bool enabled = 7; // Currently enabled for interaction
WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates
uint32 widget_id = 9; // ImGui ID (debugging / direct access)
}
message DiscoveredWindow {
string name = 1; // Window name (first segment of path)
bool visible = 2; // Whether the window is currently visible
repeated DiscoveredWidget widgets = 3; // Widgets contained in this window
}
message DiscoverWidgetsResponse {
repeated DiscoveredWindow windows = 1;
int32 total_widgets = 2; // Total number of widgets returned
int64 generated_at_ms = 3; // Snapshot timestamp (Unix ms)
}

246
src/app/core/widget_discovery_service.cc Normal file
View File

@@ -0,0 +1,246 @@
#include "app/core/widget_discovery_service.h"
#include <algorithm>
#include <map>
#include <string>
#include <utility>
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
namespace yaze {
namespace test {
namespace {
struct WindowEntry {
int index = -1;
bool visible = true;
};
} // namespace
void WidgetDiscoveryService::CollectWidgets(
ImGuiTestContext* ctx, const DiscoverWidgetsRequest& request,
DiscoverWidgetsResponse* response) const {
if (!response) {
return;
}
response->clear_windows();
response->set_total_widgets(0);
response->set_generated_at_ms(absl::ToUnixMillis(absl::Now()));
const auto& registry = gui::WidgetIdRegistry::Instance().GetAllWidgets();
if (registry.empty()) {
return;
}
const std::string window_filter_lower =
absl::AsciiStrToLower(std::string(request.window_filter()));
const std::string path_prefix_lower =
absl::AsciiStrToLower(std::string(request.path_prefix()));
const bool include_invisible = request.include_invisible();
const bool include_disabled = request.include_disabled();
std::map<std::string, WindowEntry> window_lookup;
int total_widgets = 0;
for (const auto& [path, info] : registry) {
if (!MatchesType(info.type, request.type_filter())) {
continue;
}
if (!MatchesPathPrefix(path, path_prefix_lower)) {
continue;
}
const std::string window_name = ExtractWindowName(path);
if (!MatchesWindow(window_name, window_filter_lower)) {
continue;
}
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
WindowEntry& entry = it->second;
if (inserted) {
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
if (ctx) {
ImGuiTestItemInfo window_info =
ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError);
entry.visible = (window_info.ID != 0);
}
#endif
}
if (!include_invisible && !entry.visible) {
continue;
}
if (entry.index == -1) {
DiscoveredWindow* window_proto = response->add_windows();
entry.index = response->windows_size() - 1;
window_proto->set_name(window_name);
window_proto->set_visible(entry.visible);
}
const std::string label = ExtractLabel(path);
bool widget_enabled = true;
bool widget_visible = entry.visible;
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
bool has_item_info = false;
ImGuiTestItemInfo item_info;
if (ctx) {
item_info = ctx->ItemInfo(label.c_str(), ImGuiTestOpFlags_NoError);
if (item_info.ID != 0) {
has_item_info = true;
widget_visible = item_info.RectClipped.GetWidth() > 0.0f &&
item_info.RectClipped.GetHeight() > 0.0f;
widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0;
}
}
#endif
if (!include_invisible && !widget_visible) {
continue;
}
if (!include_disabled && !widget_enabled) {
continue;
}
auto* window_proto = response->mutable_windows(entry.index);
auto* widget_proto = window_proto->add_widgets();
widget_proto->set_path(path);
widget_proto->set_label(label);
widget_proto->set_type(info.type);
widget_proto->set_suggested_action(SuggestedAction(info.type, label));
widget_proto->set_visible(widget_visible);
widget_proto->set_enabled(widget_enabled);
widget_proto->set_widget_id(info.imgui_id);
if (!info.description.empty()) {
widget_proto->set_description(info.description);
}
WidgetBounds* bounds = widget_proto->mutable_bounds();
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
if (ctx && has_item_info) {
bounds->set_min_x(item_info.RectFull.Min.x);
bounds->set_min_y(item_info.RectFull.Min.y);
bounds->set_max_x(item_info.RectFull.Max.x);
bounds->set_max_y(item_info.RectFull.Max.y);
} else {
(void)ctx;
}
#else
(void)ctx;
#endif
++total_widgets;
}
response->set_total_widgets(total_widgets);
}
bool WidgetDiscoveryService::MatchesWindow(absl::string_view window_name,
absl::string_view filter_lower) const {
if (filter_lower.empty()) {
return true;
}
std::string name_lower = absl::AsciiStrToLower(std::string(window_name));
return absl::StrContains(name_lower, filter_lower);
}
bool WidgetDiscoveryService::MatchesPathPrefix(absl::string_view path,
absl::string_view prefix_lower) const {
if (prefix_lower.empty()) {
return true;
}
std::string path_lower = absl::AsciiStrToLower(std::string(path));
return absl::StartsWith(path_lower, prefix_lower);
}
bool WidgetDiscoveryService::MatchesType(absl::string_view type,
WidgetType filter) const {
if (filter == WIDGET_TYPE_UNSPECIFIED || filter == WIDGET_TYPE_ALL) {
return true;
}
std::string type_lower = absl::AsciiStrToLower(std::string(type));
switch (filter) {
case WIDGET_TYPE_BUTTON:
return type_lower == "button" || type_lower == "menuitem";
case WIDGET_TYPE_INPUT:
return type_lower == "input" || type_lower == "textbox" ||
type_lower == "inputtext";
case WIDGET_TYPE_MENU:
return type_lower == "menu" || type_lower == "menuitem";
case WIDGET_TYPE_TAB:
return type_lower == "tab";
case WIDGET_TYPE_CHECKBOX:
return type_lower == "checkbox";
case WIDGET_TYPE_SLIDER:
return type_lower == "slider" || type_lower == "drag";
case WIDGET_TYPE_CANVAS:
return type_lower == "canvas";
case WIDGET_TYPE_SELECTABLE:
return type_lower == "selectable";
case WIDGET_TYPE_OTHER:
return true;
default:
return true;
}
}
std::string WidgetDiscoveryService::ExtractWindowName(
absl::string_view path) const {
size_t slash = path.find('/');
if (slash == absl::string_view::npos) {
return std::string(path);
}
return std::string(path.substr(0, slash));
}
std::string WidgetDiscoveryService::ExtractLabel(absl::string_view path) const {
size_t colon = path.rfind(':');
if (colon == absl::string_view::npos) {
size_t slash = path.rfind('/');
if (slash == absl::string_view::npos) {
return std::string(path);
}
return std::string(path.substr(slash + 1));
}
return std::string(path.substr(colon + 1));
}
std::string WidgetDiscoveryService::SuggestedAction(absl::string_view type,
absl::string_view label) const {
std::string type_lower = absl::AsciiStrToLower(std::string(type));
if (type_lower == "button" || type_lower == "menuitem") {
return absl::StrCat("Click button:", label);
}
if (type_lower == "checkbox") {
return absl::StrCat("Toggle checkbox:", label);
}
if (type_lower == "slider" || type_lower == "drag") {
return absl::StrCat("Adjust slider:", label);
}
if (type_lower == "input" || type_lower == "textbox" ||
type_lower == "inputtext") {
return absl::StrCat("Type text into:", label);
}
if (type_lower == "canvas") {
return absl::StrCat("Interact with canvas:", label);
}
if (type_lower == "tab") {
return absl::StrCat("Switch to tab:", label);
}
return absl::StrCat("Interact with:", label);
}
} // namespace test
} // namespace yaze

47
src/app/core/widget_discovery_service.h Normal file
View File

@@ -0,0 +1,47 @@
#ifndef YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
#define YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
#include <string>
#include <vector>
#include "absl/strings/string_view.h"
#include "app/core/proto/imgui_test_harness.pb.h"
#include "app/gui/widget_id_registry.h"
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
#include "imgui_test_engine/imgui_te_context.h"
#else
struct ImGuiTestContext;
#endif
namespace yaze {
namespace test {
// Service responsible for transforming widget registry data into
// DiscoverWidgetsResponse payloads.
class WidgetDiscoveryService {
public:
WidgetDiscoveryService() = default;
void CollectWidgets(ImGuiTestContext* ctx,
const DiscoverWidgetsRequest& request,
DiscoverWidgetsResponse* response) const;
private:
bool MatchesWindow(absl::string_view window_name,
absl::string_view filter) const;
bool MatchesPathPrefix(absl::string_view path,
absl::string_view prefix) const;
bool MatchesType(absl::string_view type,
WidgetType filter) const;
std::string ExtractWindowName(absl::string_view path) const;
std::string ExtractLabel(absl::string_view path) const;
std::string SuggestedAction(absl::string_view type,
absl::string_view label) const;
};
} // namespace test
} // namespace yaze
#endif // YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_

366
src/cli/handlers/agent.cc
View File

@@ -25,6 +25,8 @@
#include <thread>
#include <algorithm>
#include <limits>
#include <utility>
// Declare the rom flag so we can access it
ABSL_DECLARE_FLAG(std::string, rom);
@@ -105,6 +107,45 @@ std::optional<TestRunStatus> ParseStatusFilter(absl::string_view value) {
return std::nullopt;
}
std::optional<WidgetTypeFilter> ParseWidgetTypeFilter(
absl::string_view value) {
std::string lower = std::string(absl::AsciiStrToLower(value));
if (lower.empty() || lower == "unspecified" || lower == "any") {
return WidgetTypeFilter::kUnspecified;
}
if (lower == "all") {
return WidgetTypeFilter::kAll;
}
if (lower == "button" || lower == "buttons") {
return WidgetTypeFilter::kButton;
}
if (lower == "input" || lower == "textbox" || lower == "field") {
return WidgetTypeFilter::kInput;
}
if (lower == "menu" || lower == "menuitem" || lower == "menu-item") {
return WidgetTypeFilter::kMenu;
}
if (lower == "tab" || lower == "tabs") {
return WidgetTypeFilter::kTab;
}
if (lower == "checkbox" || lower == "toggle") {
return WidgetTypeFilter::kCheckbox;
}
if (lower == "slider" || lower == "drag" || lower == "sliderfloat") {
return WidgetTypeFilter::kSlider;
}
if (lower == "canvas" || lower == "viewport") {
return WidgetTypeFilter::kCanvas;
}
if (lower == "selectable" || lower == "list-item") {
return WidgetTypeFilter::kSelectable;
}
if (lower == "other") {
return WidgetTypeFilter::kOther;
}
return std::nullopt;
}
std::string HarnessAddress(const std::string& host, int port) {
return absl::StrFormat("%s:%d", host, port);
}
@@ -1051,6 +1092,327 @@ absl::Status HandleTestCommand(const std::vector<std::string>& arg_vec) {
return HandleTestRunCommand(arg_vec);
}
absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
std::string host = "localhost";
int port = 50052;
std::string window_filter;
std::string path_prefix;
std::optional<WidgetTypeFilter> type_filter;
std::optional<std::string> type_filter_label;
bool include_invisible = false;
bool include_disabled = false;
std::string format = "table";
int limit = -1;
auto require_value = [&](const std::vector<std::string>& args, size_t& index,
absl::string_view flag) -> absl::StatusOr<std::string> {
if (index + 1 >= args.size()) {
return absl::InvalidArgumentError(
absl::StrFormat("Flag %s requires a value", flag));
}
return args[++index];
};
for (size_t i = 0; i < arg_vec.size(); ++i) {
const std::string& token = arg_vec[i];
if (token == "--host") {
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--host"));
host = std::move(value);
} else if (absl::StartsWith(token, "--host=")) {
host = token.substr(7);
} else if (token == "--port") {
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--port"));
port = std::stoi(value);
} else if (absl::StartsWith(token, "--port=")) {
port = std::stoi(token.substr(7));
} else if (token == "--window" || token == "--window-filter") {
ASSIGN_OR_RETURN(auto value,
require_value(arg_vec, i, token.c_str()));
window_filter = std::move(value);
} else if (absl::StartsWith(token, "--window=")) {
window_filter = token.substr(9);
} else if (token == "--path-prefix") {
ASSIGN_OR_RETURN(auto value,
require_value(arg_vec, i, "--path-prefix"));
path_prefix = std::move(value);
} else if (absl::StartsWith(token, "--path-prefix=")) {
path_prefix = token.substr(14);
} else if (token == "--type") {
ASSIGN_OR_RETURN(auto value,
require_value(arg_vec, i, "--type"));
auto parsed = ParseWidgetTypeFilter(value);
if (!parsed.has_value()) {
return absl::InvalidArgumentError(
absl::StrFormat("Unknown widget type filter: %s", value));
}
type_filter = parsed;
type_filter_label = absl::AsciiStrToLower(value);
} else if (absl::StartsWith(token, "--type=")) {
std::string value = token.substr(7);
auto parsed = ParseWidgetTypeFilter(value);
if (!parsed.has_value()) {
return absl::InvalidArgumentError(
absl::StrFormat("Unknown widget type filter: %s", value));
}
type_filter = parsed;
type_filter_label = absl::AsciiStrToLower(value);
} else if (token == "--include-invisible") {
include_invisible = true;
} else if (token == "--include-disabled") {
include_disabled = true;
} else if (token == "--format") {
ASSIGN_OR_RETURN(auto value,
require_value(arg_vec, i, "--format"));
format = std::move(value);
} else if (absl::StartsWith(token, "--format=")) {
format = token.substr(9);
} else if (token == "--limit") {
ASSIGN_OR_RETURN(auto value,
require_value(arg_vec, i, "--limit"));
limit = std::stoi(value);
} else if (absl::StartsWith(token, "--limit=")) {
limit = std::stoi(token.substr(8));
} else if (token == "--help" || token == "-h") {
std::cout << "Usage: agent gui discover [options]\n"
<< " --host <host>\n"
<< " --port <port>\n"
<< " --window <name>\n"
<< " --type <widget-type>\n"
<< " --path-prefix <path>\n"
<< " --include-invisible\n"
<< " --include-disabled\n"
<< " --format <table|json>\n"
<< " --limit <n>\n";
return absl::OkStatus();
} else {
return absl::InvalidArgumentError(
absl::StrFormat("Unknown flag for agent gui discover: %s",
token));
}
}
format = absl::AsciiStrToLower(format);
if (format != "table" && format != "json") {
return absl::InvalidArgumentError(
"--format must be either 'table' or 'json'");
}
if (limit == 0) {
return absl::InvalidArgumentError("--limit must be positive");
}
#ifndef YAZE_WITH_GRPC
(void)host;
(void)port;
(void)window_filter;
(void)path_prefix;
(void)type_filter;
(void)include_invisible;
(void)include_disabled;
(void)format;
(void)limit;
return absl::UnimplementedError(
"GUI automation requires YAZE_WITH_GRPC=ON at build time.\n"
"Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON");
#else
GuiAutomationClient client(HarnessAddress(host, port));
RETURN_IF_ERROR(client.Connect());
DiscoverWidgetsQuery query;
query.window_filter = window_filter;
query.path_prefix = path_prefix;
if (type_filter.has_value()) {
query.type_filter = type_filter.value();
}
query.include_invisible = include_invisible;
query.include_disabled = include_disabled;
ASSIGN_OR_RETURN(auto response, client.DiscoverWidgets(query));
int max_items = limit > 0 ? limit : std::numeric_limits<int>::max();
int remaining = max_items;
std::vector<DiscoveredWindowInfo> trimmed_windows;
trimmed_windows.reserve(response.windows.size());
int rendered_widgets = 0;
for (const auto& window : response.windows) {
if (remaining <= 0) {
break;
}
DiscoveredWindowInfo trimmed;
trimmed.name = window.name;
trimmed.visible = window.visible;
for (const auto& widget : window.widgets) {
if (remaining <= 0) {
break;
}
trimmed.widgets.push_back(widget);
--remaining;
++rendered_widgets;
}
if (!trimmed.widgets.empty()) {
trimmed_windows.push_back(std::move(trimmed));
}
}
bool truncated = rendered_widgets < response.total_widgets;
if (format == "json") {
std::cout << "{\n";
std::cout << " \"server\": \""
<< JsonEscape(HarnessAddress(host, port)) << "\",\n";
std::cout << " \"totalWidgets\": " << response.total_widgets << ",\n";
std::cout << " \"returnedWidgets\": " << rendered_widgets << ",\n";
std::cout << " \"truncated\": " << (truncated ? "true" : "false")
<< ",\n";
std::cout << " \"generatedAt\": "
<< (response.generated_at.has_value()
? absl::StrCat("\"",
JsonEscape(absl::FormatTime(
"%Y-%m-%dT%H:%M:%SZ",
*response.generated_at,
absl::UTCTimeZone())),
"\"")
: std::string("null"))
<< ",\n";
std::cout << " \"windows\": [\n";
for (size_t w = 0; w < trimmed_windows.size(); ++w) {
const auto& window = trimmed_windows[w];
std::cout << " {\n";
std::cout << " \"name\": \"" << JsonEscape(window.name)
<< "\",\n";
std::cout << " \"visible\": "
<< (window.visible ? "true" : "false") << ",\n";
std::cout << " \"widgets\": [\n";
for (size_t i = 0; i < window.widgets.size(); ++i) {
const auto& widget = window.widgets[i];
std::cout << " {\n";
std::cout << " \"path\": \""
<< JsonEscape(widget.path) << "\",\n";
std::cout << " \"label\": \""
<< JsonEscape(widget.label) << "\",\n";
std::cout << " \"type\": \""
<< JsonEscape(widget.type) << "\",\n";
std::cout << " \"description\": \""
<< JsonEscape(widget.description) << "\",\n";
std::cout << " \"suggestedAction\": \""
<< JsonEscape(widget.suggested_action) << "\",\n";
std::cout << " \"visible\": "
<< (widget.visible ? "true" : "false") << ",\n";
std::cout << " \"enabled\": "
<< (widget.enabled ? "true" : "false") << ",\n";
std::cout << " \"bounds\": { \"min\": ["
<< widget.bounds.min_x << ", "
<< widget.bounds.min_y << "], \"max\": ["
<< widget.bounds.max_x << ", "
<< widget.bounds.max_y << "] },\n";
std::cout << " \"widgetId\": " << widget.widget_id
<< "\n";
std::cout << " }";
if (i + 1 < window.widgets.size()) {
std::cout << ",";
}
std::cout << "\n";
}
std::cout << " ]\n";
std::cout << " }";
if (w + 1 < trimmed_windows.size()) {
std::cout << ",";
}
std::cout << "\n";
}
std::cout << " ]\n";
std::cout << "}\n";
return absl::OkStatus();
}
std::cout << "\n=== Widget Discovery ===\n";
std::cout << "Server: " << HarnessAddress(host, port) << "\n";
if (!window_filter.empty()) {
std::cout << "Window filter: " << window_filter << "\n";
}
if (!path_prefix.empty()) {
std::cout << "Path prefix: " << path_prefix << "\n";
}
if (type_filter_label.has_value()) {
std::cout << "Type filter: " << *type_filter_label << "\n";
}
std::cout << "Include invisible: " << (include_invisible ? "yes" : "no")
<< "\n";
std::cout << "Include disabled: " << (include_disabled ? "yes" : "no")
<< "\n\n";
if (trimmed_windows.empty()) {
std::cout << "No widgets matched the provided filters." << std::endl;
return absl::OkStatus();
}
for (const auto& window : trimmed_windows) {
std::cout << "Window: " << window.name
<< (window.visible ? " (visible)" : " (hidden)")
<< "\n";
for (const auto& widget : window.widgets) {
std::cout << " • [" << widget.type << "] " << widget.label
<< "\n";
std::cout << " Path: " << widget.path << "\n";
if (!widget.description.empty()) {
std::cout << " Description: " << widget.description
<< "\n";
}
std::cout << " Suggested: " << widget.suggested_action << "\n";
std::cout << " State: "
<< (widget.visible ? "visible" : "hidden") << ", "
<< (widget.enabled ? "enabled" : "disabled") << "\n";
std::cout << absl::StrFormat(
" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n",
widget.bounds.min_x, widget.bounds.min_y,
widget.bounds.max_x, widget.bounds.max_y);
std::cout << " Widget ID: 0x" << std::hex << widget.widget_id
<< std::dec << "\n";
}
std::cout << "\n";
}
std::cout << "Widgets shown: " << rendered_widgets << " of "
<< response.total_widgets;
if (truncated) {
std::cout << " (truncated)";
}
std::cout << "\n";
if (response.generated_at.has_value()) {
std::cout << "Snapshot: "
<< absl::FormatTime("%Y-%m-%d %H:%M:%S",
*response.generated_at,
absl::LocalTimeZone())
<< "\n";
}
return absl::OkStatus();
#endif
}
absl::Status HandleGuiCommand(const std::vector<std::string>& arg_vec) {
if (arg_vec.empty()) {
return absl::InvalidArgumentError(
"Usage: agent gui <discover> [options]");
}
const std::string& subcommand = arg_vec[0];
std::vector<std::string> tail(arg_vec.begin() + 1, arg_vec.end());
if (subcommand == "discover") {
return HandleGuiDiscoverCommand(tail);
}
return absl::InvalidArgumentError(
absl::StrFormat("Unknown agent gui subcommand: %s", subcommand));
}
absl::Status HandleLearnCommand() {
std::cout << "Agent learn not yet implemented." << std::endl;
return absl::OkStatus();
@@ -1184,7 +1546,7 @@ absl::Status HandleDescribeCommand(const std::vector<std::string>& arg_vec) {
absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
if (arg_vec.empty()) {
return absl::InvalidArgumentError(
"Usage: agent <run|plan|diff|test|learn|list|commit|revert|describe> [options]");
"Usage: agent <run|plan|diff|test|gui|learn|list|commit|revert|describe> [options]");
}
std::string subcommand = arg_vec[0];
@@ -1198,6 +1560,8 @@ absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
return HandleDiffCommand(rom_, subcommand_args);
} else if (subcommand == "test") {
return HandleTestCommand(subcommand_args);
} else if (subcommand == "gui") {
return HandleGuiCommand(subcommand_args);
} else if (subcommand == "learn") {
return HandleLearnCommand();
} else if (subcommand == "list") {

4
src/cli/modern_cli.cc
View File

@@ -60,11 +60,13 @@ void ModernCLI::SetupCommands() {
commands_["agent"] = {
.name = "agent",
.description = "Interact with the AI agent",
.usage = "z3ed agent <run|plan|diff|test|list|learn|commit|revert|describe> [options]\n"
.usage = "z3ed agent <run|plan|diff|test|gui|list|learn|commit|revert|describe> [options]\n"
" test run: --prompt \"<description>\" [--host <host>] [--port <port>] [--timeout <sec>]\n"
" test status: status --test-id <id> [--follow] [--host <host>] [--port <port>]\n"
" test list: list [--category <name>] [--status <state>] [--limit <n>] [--host <host>] [--port <port>]\n"
" test results: results --test-id <id> [--include-logs] [--format yaml|json] [--host <host>] [--port <port>]\n"
" gui discover: discover [--window <name>] [--type <widget>] [--path-prefix <path>]\n"
" [--include-invisible] [--include-disabled] [--format table|json] [--limit <n>]\n"
" describe options: [--resource <name>] [--format json|yaml] [--output <path>]\n"
" [--version <value>] [--last-updated <date>]",
.handler = [this](const std::vector<std::string>& args) -> absl::Status {

97
src/cli/service/gui_automation_client.cc
View File

@@ -21,6 +21,35 @@ std::optional<absl::Time> OptionalTimeFromMillis(int64_t millis) {
return absl::FromUnixMillis(millis);
}
yaze::test::WidgetType ConvertWidgetTypeFilterToProto(WidgetTypeFilter filter) {
using ProtoType = yaze::test::WidgetType;
switch (filter) {
case WidgetTypeFilter::kAll:
return ProtoType::WIDGET_TYPE_ALL;
case WidgetTypeFilter::kButton:
return ProtoType::WIDGET_TYPE_BUTTON;
case WidgetTypeFilter::kInput:
return ProtoType::WIDGET_TYPE_INPUT;
case WidgetTypeFilter::kMenu:
return ProtoType::WIDGET_TYPE_MENU;
case WidgetTypeFilter::kTab:
return ProtoType::WIDGET_TYPE_TAB;
case WidgetTypeFilter::kCheckbox:
return ProtoType::WIDGET_TYPE_CHECKBOX;
case WidgetTypeFilter::kSlider:
return ProtoType::WIDGET_TYPE_SLIDER;
case WidgetTypeFilter::kCanvas:
return ProtoType::WIDGET_TYPE_CANVAS;
case WidgetTypeFilter::kSelectable:
return ProtoType::WIDGET_TYPE_SELECTABLE;
case WidgetTypeFilter::kOther:
return ProtoType::WIDGET_TYPE_OTHER;
case WidgetTypeFilter::kUnspecified:
default:
return ProtoType::WIDGET_TYPE_UNSPECIFIED;
}
}
TestRunStatus ConvertStatusProto(
yaze::test::GetTestStatusResponse::Status status) {
using ProtoStatus = yaze::test::GetTestStatusResponse::Status;
@@ -440,5 +469,73 @@ absl::StatusOr<TestResultDetails> GuiAutomationClient::GetTestResults(
#endif
}
absl::StatusOr<DiscoverWidgetsResult> GuiAutomationClient::DiscoverWidgets(
const DiscoverWidgetsQuery& query) {
#ifdef YAZE_WITH_GRPC
if (!stub_) {
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
}
yaze::test::DiscoverWidgetsRequest request;
if (!query.window_filter.empty()) {
request.set_window_filter(query.window_filter);
}
request.set_type_filter(ConvertWidgetTypeFilterToProto(query.type_filter));
if (!query.path_prefix.empty()) {
request.set_path_prefix(query.path_prefix);
}
request.set_include_invisible(query.include_invisible);
request.set_include_disabled(query.include_disabled);
yaze::test::DiscoverWidgetsResponse response;
grpc::ClientContext context;
grpc::Status status = stub_->DiscoverWidgets(&context, request, &response);
if (!status.ok()) {
return absl::InternalError(
absl::StrFormat("DiscoverWidgets RPC failed: %s",
status.error_message()));
}
DiscoverWidgetsResult result;
result.total_widgets = response.total_widgets();
if (response.generated_at_ms() > 0) {
result.generated_at = OptionalTimeFromMillis(response.generated_at_ms());
}
result.windows.reserve(response.windows_size());
for (const auto& window_proto : response.windows()) {
DiscoveredWindowInfo window_info;
window_info.name = window_proto.name();
window_info.visible = window_proto.visible();
window_info.widgets.reserve(window_proto.widgets_size());
for (const auto& widget_proto : window_proto.widgets()) {
WidgetDescriptor widget;
widget.path = widget_proto.path();
widget.label = widget_proto.label();
widget.type = widget_proto.type();
widget.description = widget_proto.description();
widget.suggested_action = widget_proto.suggested_action();
widget.visible = widget_proto.visible();
widget.enabled = widget_proto.enabled();
widget.bounds.min_x = widget_proto.bounds().min_x();
widget.bounds.min_y = widget_proto.bounds().min_y();
widget.bounds.max_x = widget_proto.bounds().max_x();
widget.bounds.max_y = widget_proto.bounds().max_y();
widget.widget_id = widget_proto.widget_id();
window_info.widgets.push_back(std::move(widget));
}
result.windows.push_back(std::move(window_info));
}
return result;
#else
(void)query;
return absl::UnimplementedError("gRPC not available");
#endif
}
} // namespace cli
} // namespace yaze

57
src/cli/service/gui_automation_client.h
View File

@@ -9,6 +9,7 @@
#include "absl/time/time.h"
#include <chrono>
#include <cstdint>
#include <map>
#include <memory>
#include <optional>
@@ -120,6 +121,59 @@ struct TestResultDetails {
std::map<std::string, int> metrics;
};
enum class WidgetTypeFilter {
kUnspecified,
kAll,
kButton,
kInput,
kMenu,
kTab,
kCheckbox,
kSlider,
kCanvas,
kSelectable,
kOther,
};
struct WidgetBoundingBox {
float min_x = 0.0f;
float min_y = 0.0f;
float max_x = 0.0f;
float max_y = 0.0f;
};
struct WidgetDescriptor {
std::string path;
std::string label;
std::string type;
std::string description;
std::string suggested_action;
bool visible = true;
bool enabled = true;
WidgetBoundingBox bounds;
uint32_t widget_id = 0;
};
struct DiscoveredWindowInfo {
std::string name;
bool visible = true;
std::vector<WidgetDescriptor> widgets;
};
struct DiscoverWidgetsQuery {
std::string window_filter;
WidgetTypeFilter type_filter = WidgetTypeFilter::kUnspecified;
std::string path_prefix;
bool include_invisible = false;
bool include_disabled = false;
};
struct DiscoverWidgetsResult {
std::vector<DiscoveredWindowInfo> windows;
int total_widgets = 0;
std::optional<absl::Time> generated_at;
};
/**
* @brief Client for automating YAZE GUI through gRPC
*
@@ -225,6 +279,9 @@ class GuiAutomationClient {
absl::StatusOr<TestResultDetails> GetTestResults(const std::string& test_id,
bool include_logs = false);
absl::StatusOr<DiscoverWidgetsResult> DiscoverWidgets(
const DiscoverWidgetsQuery& query);
/**
* @brief Check if client is connected
*/