feat: Implement widget discovery feature in GUI automation
- Added `DiscoverWidgets` RPC to the ImGuiTestHarness service for enumerating GUI widgets. - Introduced `WidgetDiscoveryService` to handle widget collection and filtering based on various criteria. - Updated `agent gui discover` command to support new options for filtering and output formats. - Enhanced `GuiAutomationClient` to facilitate widget discovery requests and responses. - Added necessary protobuf messages for widget discovery in `imgui_test_harness.proto`. - Updated CLI command handling to include new GUI discovery functionality. - Improved documentation for the `agent gui discover` command with examples and output formats.
This commit is contained in:
scawful
@@ -25,6 +25,8 @@
|
||||
#include <thread>
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
|
||||
// Declare the rom flag so we can access it
|
||||
ABSL_DECLARE_FLAG(std::string, rom);
|
||||
@@ -105,6 +107,45 @@ std::optional<TestRunStatus> ParseStatusFilter(absl::string_view value) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<WidgetTypeFilter> ParseWidgetTypeFilter(
|
||||
absl::string_view value) {
|
||||
std::string lower = std::string(absl::AsciiStrToLower(value));
|
||||
if (lower.empty() || lower == "unspecified" || lower == "any") {
|
||||
return WidgetTypeFilter::kUnspecified;
|
||||
}
|
||||
if (lower == "all") {
|
||||
return WidgetTypeFilter::kAll;
|
||||
}
|
||||
if (lower == "button" || lower == "buttons") {
|
||||
return WidgetTypeFilter::kButton;
|
||||
}
|
||||
if (lower == "input" || lower == "textbox" || lower == "field") {
|
||||
return WidgetTypeFilter::kInput;
|
||||
}
|
||||
if (lower == "menu" || lower == "menuitem" || lower == "menu-item") {
|
||||
return WidgetTypeFilter::kMenu;
|
||||
}
|
||||
if (lower == "tab" || lower == "tabs") {
|
||||
return WidgetTypeFilter::kTab;
|
||||
}
|
||||
if (lower == "checkbox" || lower == "toggle") {
|
||||
return WidgetTypeFilter::kCheckbox;
|
||||
}
|
||||
if (lower == "slider" || lower == "drag" || lower == "sliderfloat") {
|
||||
return WidgetTypeFilter::kSlider;
|
||||
}
|
||||
if (lower == "canvas" || lower == "viewport") {
|
||||
return WidgetTypeFilter::kCanvas;
|
||||
}
|
||||
if (lower == "selectable" || lower == "list-item") {
|
||||
return WidgetTypeFilter::kSelectable;
|
||||
}
|
||||
if (lower == "other") {
|
||||
return WidgetTypeFilter::kOther;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::string HarnessAddress(const std::string& host, int port) {
|
||||
return absl::StrFormat("%s:%d", host, port);
|
||||
}
|
||||
@@ -1051,6 +1092,327 @@ absl::Status HandleTestCommand(const std::vector<std::string>& arg_vec) {
|
||||
return HandleTestRunCommand(arg_vec);
|
||||
}
|
||||
|
||||
absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
|
||||
std::string host = "localhost";
|
||||
int port = 50052;
|
||||
std::string window_filter;
|
||||
std::string path_prefix;
|
||||
std::optional<WidgetTypeFilter> type_filter;
|
||||
std::optional<std::string> type_filter_label;
|
||||
bool include_invisible = false;
|
||||
bool include_disabled = false;
|
||||
std::string format = "table";
|
||||
int limit = -1;
|
||||
|
||||
auto require_value = [&](const std::vector<std::string>& args, size_t& index,
|
||||
absl::string_view flag) -> absl::StatusOr<std::string> {
|
||||
if (index + 1 >= args.size()) {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Flag %s requires a value", flag));
|
||||
}
|
||||
return args[++index];
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < arg_vec.size(); ++i) {
|
||||
const std::string& token = arg_vec[i];
|
||||
if (token == "--host") {
|
||||
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--host"));
|
||||
host = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--host=")) {
|
||||
host = token.substr(7);
|
||||
} else if (token == "--port") {
|
||||
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--port"));
|
||||
port = std::stoi(value);
|
||||
} else if (absl::StartsWith(token, "--port=")) {
|
||||
port = std::stoi(token.substr(7));
|
||||
} else if (token == "--window" || token == "--window-filter") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, token.c_str()));
|
||||
window_filter = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--window=")) {
|
||||
window_filter = token.substr(9);
|
||||
} else if (token == "--path-prefix") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--path-prefix"));
|
||||
path_prefix = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--path-prefix=")) {
|
||||
path_prefix = token.substr(14);
|
||||
} else if (token == "--type") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--type"));
|
||||
auto parsed = ParseWidgetTypeFilter(value);
|
||||
if (!parsed.has_value()) {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown widget type filter: %s", value));
|
||||
}
|
||||
type_filter = parsed;
|
||||
type_filter_label = absl::AsciiStrToLower(value);
|
||||
} else if (absl::StartsWith(token, "--type=")) {
|
||||
std::string value = token.substr(7);
|
||||
auto parsed = ParseWidgetTypeFilter(value);
|
||||
if (!parsed.has_value()) {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown widget type filter: %s", value));
|
||||
}
|
||||
type_filter = parsed;
|
||||
type_filter_label = absl::AsciiStrToLower(value);
|
||||
} else if (token == "--include-invisible") {
|
||||
include_invisible = true;
|
||||
} else if (token == "--include-disabled") {
|
||||
include_disabled = true;
|
||||
} else if (token == "--format") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--format"));
|
||||
format = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--format=")) {
|
||||
format = token.substr(9);
|
||||
} else if (token == "--limit") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--limit"));
|
||||
limit = std::stoi(value);
|
||||
} else if (absl::StartsWith(token, "--limit=")) {
|
||||
limit = std::stoi(token.substr(8));
|
||||
} else if (token == "--help" || token == "-h") {
|
||||
std::cout << "Usage: agent gui discover [options]\n"
|
||||
<< " --host <host>\n"
|
||||
<< " --port <port>\n"
|
||||
<< " --window <name>\n"
|
||||
<< " --type <widget-type>\n"
|
||||
<< " --path-prefix <path>\n"
|
||||
<< " --include-invisible\n"
|
||||
<< " --include-disabled\n"
|
||||
<< " --format <table|json>\n"
|
||||
<< " --limit <n>\n";
|
||||
return absl::OkStatus();
|
||||
} else {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown flag for agent gui discover: %s",
|
||||
token));
|
||||
}
|
||||
}
|
||||
|
||||
format = absl::AsciiStrToLower(format);
|
||||
if (format != "table" && format != "json") {
|
||||
return absl::InvalidArgumentError(
|
||||
"--format must be either 'table' or 'json'");
|
||||
}
|
||||
|
||||
if (limit == 0) {
|
||||
return absl::InvalidArgumentError("--limit must be positive");
|
||||
}
|
||||
|
||||
#ifndef YAZE_WITH_GRPC
|
||||
(void)host;
|
||||
(void)port;
|
||||
(void)window_filter;
|
||||
(void)path_prefix;
|
||||
(void)type_filter;
|
||||
(void)include_invisible;
|
||||
(void)include_disabled;
|
||||
(void)format;
|
||||
(void)limit;
|
||||
return absl::UnimplementedError(
|
||||
"GUI automation requires YAZE_WITH_GRPC=ON at build time.\n"
|
||||
"Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON");
|
||||
#else
|
||||
GuiAutomationClient client(HarnessAddress(host, port));
|
||||
RETURN_IF_ERROR(client.Connect());
|
||||
|
||||
DiscoverWidgetsQuery query;
|
||||
query.window_filter = window_filter;
|
||||
query.path_prefix = path_prefix;
|
||||
if (type_filter.has_value()) {
|
||||
query.type_filter = type_filter.value();
|
||||
}
|
||||
query.include_invisible = include_invisible;
|
||||
query.include_disabled = include_disabled;
|
||||
|
||||
ASSIGN_OR_RETURN(auto response, client.DiscoverWidgets(query));
|
||||
|
||||
int max_items = limit > 0 ? limit : std::numeric_limits<int>::max();
|
||||
int remaining = max_items;
|
||||
std::vector<DiscoveredWindowInfo> trimmed_windows;
|
||||
trimmed_windows.reserve(response.windows.size());
|
||||
int rendered_widgets = 0;
|
||||
|
||||
for (const auto& window : response.windows) {
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
DiscoveredWindowInfo trimmed;
|
||||
trimmed.name = window.name;
|
||||
trimmed.visible = window.visible;
|
||||
|
||||
for (const auto& widget : window.widgets) {
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
trimmed.widgets.push_back(widget);
|
||||
--remaining;
|
||||
++rendered_widgets;
|
||||
}
|
||||
|
||||
if (!trimmed.widgets.empty()) {
|
||||
trimmed_windows.push_back(std::move(trimmed));
|
||||
}
|
||||
}
|
||||
|
||||
bool truncated = rendered_widgets < response.total_widgets;
|
||||
|
||||
if (format == "json") {
|
||||
std::cout << "{\n";
|
||||
std::cout << " \"server\": \""
|
||||
<< JsonEscape(HarnessAddress(host, port)) << "\",\n";
|
||||
std::cout << " \"totalWidgets\": " << response.total_widgets << ",\n";
|
||||
std::cout << " \"returnedWidgets\": " << rendered_widgets << ",\n";
|
||||
std::cout << " \"truncated\": " << (truncated ? "true" : "false")
|
||||
<< ",\n";
|
||||
std::cout << " \"generatedAt\": "
|
||||
<< (response.generated_at.has_value()
|
||||
? absl::StrCat("\"",
|
||||
JsonEscape(absl::FormatTime(
|
||||
"%Y-%m-%dT%H:%M:%SZ",
|
||||
*response.generated_at,
|
||||
absl::UTCTimeZone())),
|
||||
"\"")
|
||||
: std::string("null"))
|
||||
<< ",\n";
|
||||
std::cout << " \"windows\": [\n";
|
||||
|
||||
for (size_t w = 0; w < trimmed_windows.size(); ++w) {
|
||||
const auto& window = trimmed_windows[w];
|
||||
std::cout << " {\n";
|
||||
std::cout << " \"name\": \"" << JsonEscape(window.name)
|
||||
<< "\",\n";
|
||||
std::cout << " \"visible\": "
|
||||
<< (window.visible ? "true" : "false") << ",\n";
|
||||
std::cout << " \"widgets\": [\n";
|
||||
for (size_t i = 0; i < window.widgets.size(); ++i) {
|
||||
const auto& widget = window.widgets[i];
|
||||
std::cout << " {\n";
|
||||
std::cout << " \"path\": \""
|
||||
<< JsonEscape(widget.path) << "\",\n";
|
||||
std::cout << " \"label\": \""
|
||||
<< JsonEscape(widget.label) << "\",\n";
|
||||
std::cout << " \"type\": \""
|
||||
<< JsonEscape(widget.type) << "\",\n";
|
||||
std::cout << " \"description\": \""
|
||||
<< JsonEscape(widget.description) << "\",\n";
|
||||
std::cout << " \"suggestedAction\": \""
|
||||
<< JsonEscape(widget.suggested_action) << "\",\n";
|
||||
std::cout << " \"visible\": "
|
||||
<< (widget.visible ? "true" : "false") << ",\n";
|
||||
std::cout << " \"enabled\": "
|
||||
<< (widget.enabled ? "true" : "false") << ",\n";
|
||||
std::cout << " \"bounds\": { \"min\": ["
|
||||
<< widget.bounds.min_x << ", "
|
||||
<< widget.bounds.min_y << "], \"max\": ["
|
||||
<< widget.bounds.max_x << ", "
|
||||
<< widget.bounds.max_y << "] },\n";
|
||||
std::cout << " \"widgetId\": " << widget.widget_id
|
||||
<< "\n";
|
||||
std::cout << " }";
|
||||
if (i + 1 < window.widgets.size()) {
|
||||
std::cout << ",";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
std::cout << " ]\n";
|
||||
std::cout << " }";
|
||||
if (w + 1 < trimmed_windows.size()) {
|
||||
std::cout << ",";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
|
||||
std::cout << " ]\n";
|
||||
std::cout << "}\n";
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
std::cout << "\n=== Widget Discovery ===\n";
|
||||
std::cout << "Server: " << HarnessAddress(host, port) << "\n";
|
||||
if (!window_filter.empty()) {
|
||||
std::cout << "Window filter: " << window_filter << "\n";
|
||||
}
|
||||
if (!path_prefix.empty()) {
|
||||
std::cout << "Path prefix: " << path_prefix << "\n";
|
||||
}
|
||||
if (type_filter_label.has_value()) {
|
||||
std::cout << "Type filter: " << *type_filter_label << "\n";
|
||||
}
|
||||
std::cout << "Include invisible: " << (include_invisible ? "yes" : "no")
|
||||
<< "\n";
|
||||
std::cout << "Include disabled: " << (include_disabled ? "yes" : "no")
|
||||
<< "\n\n";
|
||||
|
||||
if (trimmed_windows.empty()) {
|
||||
std::cout << "No widgets matched the provided filters." << std::endl;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
for (const auto& window : trimmed_windows) {
|
||||
std::cout << "Window: " << window.name
|
||||
<< (window.visible ? " (visible)" : " (hidden)")
|
||||
<< "\n";
|
||||
for (const auto& widget : window.widgets) {
|
||||
std::cout << " • [" << widget.type << "] " << widget.label
|
||||
<< "\n";
|
||||
std::cout << " Path: " << widget.path << "\n";
|
||||
if (!widget.description.empty()) {
|
||||
std::cout << " Description: " << widget.description
|
||||
<< "\n";
|
||||
}
|
||||
std::cout << " Suggested: " << widget.suggested_action << "\n";
|
||||
std::cout << " State: "
|
||||
<< (widget.visible ? "visible" : "hidden") << ", "
|
||||
<< (widget.enabled ? "enabled" : "disabled") << "\n";
|
||||
std::cout << absl::StrFormat(
|
||||
" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n",
|
||||
widget.bounds.min_x, widget.bounds.min_y,
|
||||
widget.bounds.max_x, widget.bounds.max_y);
|
||||
std::cout << " Widget ID: 0x" << std::hex << widget.widget_id
|
||||
<< std::dec << "\n";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
|
||||
std::cout << "Widgets shown: " << rendered_widgets << " of "
|
||||
<< response.total_widgets;
|
||||
if (truncated) {
|
||||
std::cout << " (truncated)";
|
||||
}
|
||||
std::cout << "\n";
|
||||
|
||||
if (response.generated_at.has_value()) {
|
||||
std::cout << "Snapshot: "
|
||||
<< absl::FormatTime("%Y-%m-%d %H:%M:%S",
|
||||
*response.generated_at,
|
||||
absl::LocalTimeZone())
|
||||
<< "\n";
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::Status HandleGuiCommand(const std::vector<std::string>& arg_vec) {
|
||||
if (arg_vec.empty()) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Usage: agent gui <discover> [options]");
|
||||
}
|
||||
|
||||
const std::string& subcommand = arg_vec[0];
|
||||
std::vector<std::string> tail(arg_vec.begin() + 1, arg_vec.end());
|
||||
|
||||
if (subcommand == "discover") {
|
||||
return HandleGuiDiscoverCommand(tail);
|
||||
}
|
||||
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown agent gui subcommand: %s", subcommand));
|
||||
}
|
||||
|
||||
absl::Status HandleLearnCommand() {
|
||||
std::cout << "Agent learn not yet implemented." << std::endl;
|
||||
return absl::OkStatus();
|
||||
@@ -1184,7 +1546,7 @@ absl::Status HandleDescribeCommand(const std::vector<std::string>& arg_vec) {
|
||||
absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
|
||||
if (arg_vec.empty()) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Usage: agent <run|plan|diff|test|learn|list|commit|revert|describe> [options]");
|
||||
"Usage: agent <run|plan|diff|test|gui|learn|list|commit|revert|describe> [options]");
|
||||
}
|
||||
|
||||
std::string subcommand = arg_vec[0];
|
||||
@@ -1198,6 +1560,8 @@ absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
|
||||
return HandleDiffCommand(rom_, subcommand_args);
|
||||
} else if (subcommand == "test") {
|
||||
return HandleTestCommand(subcommand_args);
|
||||
} else if (subcommand == "gui") {
|
||||
return HandleGuiCommand(subcommand_args);
|
||||
} else if (subcommand == "learn") {
|
||||
return HandleLearnCommand();
|
||||
} else if (subcommand == "list") {
|
||||
|
||||
@@ -60,11 +60,13 @@ void ModernCLI::SetupCommands() {
|
||||
commands_["agent"] = {
|
||||
.name = "agent",
|
||||
.description = "Interact with the AI agent",
|
||||
.usage = "z3ed agent <run|plan|diff|test|list|learn|commit|revert|describe> [options]\n"
|
||||
.usage = "z3ed agent <run|plan|diff|test|gui|list|learn|commit|revert|describe> [options]\n"
|
||||
" test run: --prompt \"<description>\" [--host <host>] [--port <port>] [--timeout <sec>]\n"
|
||||
" test status: status --test-id <id> [--follow] [--host <host>] [--port <port>]\n"
|
||||
" test list: list [--category <name>] [--status <state>] [--limit <n>] [--host <host>] [--port <port>]\n"
|
||||
" test results: results --test-id <id> [--include-logs] [--format yaml|json] [--host <host>] [--port <port>]\n"
|
||||
" gui discover: discover [--window <name>] [--type <widget>] [--path-prefix <path>]\n"
|
||||
" [--include-invisible] [--include-disabled] [--format table|json] [--limit <n>]\n"
|
||||
" describe options: [--resource <name>] [--format json|yaml] [--output <path>]\n"
|
||||
" [--version <value>] [--last-updated <date>]",
|
||||
.handler = [this](const std::vector<std::string>& args) -> absl::Status {
|
||||
|
||||
@@ -21,6 +21,35 @@ std::optional<absl::Time> OptionalTimeFromMillis(int64_t millis) {
|
||||
return absl::FromUnixMillis(millis);
|
||||
}
|
||||
|
||||
yaze::test::WidgetType ConvertWidgetTypeFilterToProto(WidgetTypeFilter filter) {
|
||||
using ProtoType = yaze::test::WidgetType;
|
||||
switch (filter) {
|
||||
case WidgetTypeFilter::kAll:
|
||||
return ProtoType::WIDGET_TYPE_ALL;
|
||||
case WidgetTypeFilter::kButton:
|
||||
return ProtoType::WIDGET_TYPE_BUTTON;
|
||||
case WidgetTypeFilter::kInput:
|
||||
return ProtoType::WIDGET_TYPE_INPUT;
|
||||
case WidgetTypeFilter::kMenu:
|
||||
return ProtoType::WIDGET_TYPE_MENU;
|
||||
case WidgetTypeFilter::kTab:
|
||||
return ProtoType::WIDGET_TYPE_TAB;
|
||||
case WidgetTypeFilter::kCheckbox:
|
||||
return ProtoType::WIDGET_TYPE_CHECKBOX;
|
||||
case WidgetTypeFilter::kSlider:
|
||||
return ProtoType::WIDGET_TYPE_SLIDER;
|
||||
case WidgetTypeFilter::kCanvas:
|
||||
return ProtoType::WIDGET_TYPE_CANVAS;
|
||||
case WidgetTypeFilter::kSelectable:
|
||||
return ProtoType::WIDGET_TYPE_SELECTABLE;
|
||||
case WidgetTypeFilter::kOther:
|
||||
return ProtoType::WIDGET_TYPE_OTHER;
|
||||
case WidgetTypeFilter::kUnspecified:
|
||||
default:
|
||||
return ProtoType::WIDGET_TYPE_UNSPECIFIED;
|
||||
}
|
||||
}
|
||||
|
||||
TestRunStatus ConvertStatusProto(
|
||||
yaze::test::GetTestStatusResponse::Status status) {
|
||||
using ProtoStatus = yaze::test::GetTestStatusResponse::Status;
|
||||
@@ -440,5 +469,73 @@ absl::StatusOr<TestResultDetails> GuiAutomationClient::GetTestResults(
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<DiscoverWidgetsResult> GuiAutomationClient::DiscoverWidgets(
|
||||
const DiscoverWidgetsQuery& query) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::DiscoverWidgetsRequest request;
|
||||
if (!query.window_filter.empty()) {
|
||||
request.set_window_filter(query.window_filter);
|
||||
}
|
||||
request.set_type_filter(ConvertWidgetTypeFilterToProto(query.type_filter));
|
||||
if (!query.path_prefix.empty()) {
|
||||
request.set_path_prefix(query.path_prefix);
|
||||
}
|
||||
request.set_include_invisible(query.include_invisible);
|
||||
request.set_include_disabled(query.include_disabled);
|
||||
|
||||
yaze::test::DiscoverWidgetsResponse response;
|
||||
grpc::ClientContext context;
|
||||
grpc::Status status = stub_->DiscoverWidgets(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("DiscoverWidgets RPC failed: %s",
|
||||
status.error_message()));
|
||||
}
|
||||
|
||||
DiscoverWidgetsResult result;
|
||||
result.total_widgets = response.total_widgets();
|
||||
if (response.generated_at_ms() > 0) {
|
||||
result.generated_at = OptionalTimeFromMillis(response.generated_at_ms());
|
||||
}
|
||||
|
||||
result.windows.reserve(response.windows_size());
|
||||
for (const auto& window_proto : response.windows()) {
|
||||
DiscoveredWindowInfo window_info;
|
||||
window_info.name = window_proto.name();
|
||||
window_info.visible = window_proto.visible();
|
||||
window_info.widgets.reserve(window_proto.widgets_size());
|
||||
|
||||
for (const auto& widget_proto : window_proto.widgets()) {
|
||||
WidgetDescriptor widget;
|
||||
widget.path = widget_proto.path();
|
||||
widget.label = widget_proto.label();
|
||||
widget.type = widget_proto.type();
|
||||
widget.description = widget_proto.description();
|
||||
widget.suggested_action = widget_proto.suggested_action();
|
||||
widget.visible = widget_proto.visible();
|
||||
widget.enabled = widget_proto.enabled();
|
||||
widget.bounds.min_x = widget_proto.bounds().min_x();
|
||||
widget.bounds.min_y = widget_proto.bounds().min_y();
|
||||
widget.bounds.max_x = widget_proto.bounds().max_x();
|
||||
widget.bounds.max_y = widget_proto.bounds().max_y();
|
||||
widget.widget_id = widget_proto.widget_id();
|
||||
window_info.widgets.push_back(std::move(widget));
|
||||
}
|
||||
|
||||
result.windows.push_back(std::move(window_info));
|
||||
}
|
||||
|
||||
return result;
|
||||
#else
|
||||
(void)query;
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "absl/time/time.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
@@ -120,6 +121,59 @@ struct TestResultDetails {
|
||||
std::map<std::string, int> metrics;
|
||||
};
|
||||
|
||||
enum class WidgetTypeFilter {
|
||||
kUnspecified,
|
||||
kAll,
|
||||
kButton,
|
||||
kInput,
|
||||
kMenu,
|
||||
kTab,
|
||||
kCheckbox,
|
||||
kSlider,
|
||||
kCanvas,
|
||||
kSelectable,
|
||||
kOther,
|
||||
};
|
||||
|
||||
struct WidgetBoundingBox {
|
||||
float min_x = 0.0f;
|
||||
float min_y = 0.0f;
|
||||
float max_x = 0.0f;
|
||||
float max_y = 0.0f;
|
||||
};
|
||||
|
||||
struct WidgetDescriptor {
|
||||
std::string path;
|
||||
std::string label;
|
||||
std::string type;
|
||||
std::string description;
|
||||
std::string suggested_action;
|
||||
bool visible = true;
|
||||
bool enabled = true;
|
||||
WidgetBoundingBox bounds;
|
||||
uint32_t widget_id = 0;
|
||||
};
|
||||
|
||||
struct DiscoveredWindowInfo {
|
||||
std::string name;
|
||||
bool visible = true;
|
||||
std::vector<WidgetDescriptor> widgets;
|
||||
};
|
||||
|
||||
struct DiscoverWidgetsQuery {
|
||||
std::string window_filter;
|
||||
WidgetTypeFilter type_filter = WidgetTypeFilter::kUnspecified;
|
||||
std::string path_prefix;
|
||||
bool include_invisible = false;
|
||||
bool include_disabled = false;
|
||||
};
|
||||
|
||||
struct DiscoverWidgetsResult {
|
||||
std::vector<DiscoveredWindowInfo> windows;
|
||||
int total_widgets = 0;
|
||||
std::optional<absl::Time> generated_at;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Client for automating YAZE GUI through gRPC
|
||||
*
|
||||
@@ -225,6 +279,9 @@ class GuiAutomationClient {
|
||||
absl::StatusOr<TestResultDetails> GetTestResults(const std::string& test_id,
|
||||
bool include_logs = false);
|
||||
|
||||
absl::StatusOr<DiscoverWidgetsResult> DiscoverWidgets(
|
||||
const DiscoverWidgetsQuery& query);
|
||||
|
||||
/**
|
||||
* @brief Check if client is connected
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user