feat: Implement widget discovery feature in GUI automation
- Added `DiscoverWidgets` RPC to the ImGuiTestHarness service for enumerating GUI widgets. - Introduced `WidgetDiscoveryService` to handle widget collection and filtering based on various criteria. - Updated `agent gui discover` command to support new options for filtering and output formats. - Enhanced `GuiAutomationClient` to facilitate widget discovery requests and responses. - Added necessary protobuf messages for widget discovery in `imgui_test_harness.proto`. - Updated CLI command handling to include new GUI discovery functionality. - Improved documentation for the `agent gui discover` command with examples and output formats.
This commit is contained in:
@@ -248,44 +248,54 @@ Examples:
|
|||||||
|
|
||||||
##### `agent gui discover` - Enumerate available widgets
|
##### `agent gui discover` - Enumerate available widgets
|
||||||
```bash
|
```bash
|
||||||
z3ed agent gui discover [--window <name>] [--type <widget_type>] [--format <json|yaml>]
|
z3ed agent gui discover \
|
||||||
|
[--host <name>] [--port <port>] \
|
||||||
|
[--window <name>] [--path-prefix <path>] \
|
||||||
|
[--type <widget_type>] [--include-invisible] [--include-disabled] \
|
||||||
|
[--format <table|json>] [--limit <n>]
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
--window <name> Filter by window name (e.g. "Overworld")
|
--host <name> Harness host (default: localhost)
|
||||||
--type <type> Filter by widget type: button, input, menu, tab, checkbox
|
--port <port> Harness port (default: 50052)
|
||||||
--format <format> Output format: json or yaml (default: yaml)
|
--window <name> Filter by window name (case-insensitive substring)
|
||||||
|
--path-prefix <path> Require widget path to start with prefix
|
||||||
|
--type <type> Filter widget type: button, input, menu, tab,
|
||||||
|
checkbox, slider, canvas, selectable, other
|
||||||
|
--include-invisible Include widgets whose parent window is hidden
|
||||||
|
--include-disabled Include widgets flagged as disabled
|
||||||
|
--format <mode> Output as `table` (default) or `json`
|
||||||
|
--limit <n> Maximum widgets to display (useful for large UIs)
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
# Discover all widgets
|
# Discover all widgets currently registered
|
||||||
z3ed agent gui discover
|
z3ed agent gui discover
|
||||||
|
|
||||||
# Find all buttons in Overworld editor
|
# Focus on buttons inside the Overworld editor window
|
||||||
z3ed agent gui discover --window "Overworld" --type button
|
z3ed agent gui discover --window "Overworld" --type button
|
||||||
|
|
||||||
# Get JSON for AI consumption
|
# Export a JSON snapshot for an automation agent (showing first 50 widgets)
|
||||||
z3ed agent gui discover --format json > widgets.json
|
z3ed agent gui discover --format json --limit 50 > widgets.json
|
||||||
```
|
```
|
||||||
|
|
||||||
**Output Example**:
|
**Table Output Example**:
|
||||||
```yaml
|
```
|
||||||
windows:
|
=== Widget Discovery ===
|
||||||
- name: Main Window
|
Server: localhost:50052
|
||||||
visible: true
|
Window filter: Overworld
|
||||||
widgets:
|
Type filter: button
|
||||||
- id: menu_file
|
Include invisible: no
|
||||||
label: File
|
Include disabled: no
|
||||||
type: menu
|
|
||||||
enabled: true
|
Window: Overworld (visible)
|
||||||
suggested_action: "Click menuitem: File"
|
• [button] Save
|
||||||
- name: Overworld
|
Path: Overworld/Toolbar/button:Save
|
||||||
visible: true
|
Suggested: Click button:Save
|
||||||
widgets:
|
State: visible, enabled
|
||||||
- id: btn_save
|
Bounds: (24.0, 64.0) → (112.0, 92.0)
|
||||||
label: Save
|
Widget ID: 0x13fc41a2
|
||||||
type: button
|
|
||||||
enabled: true
|
Widgets shown: 3 of 18 (truncated)
|
||||||
position: "10,20,100,30"
|
Snapshot: 2025-01-16 19:42:05
|
||||||
suggested_action: "Click button:Save"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Use Cases**:
|
**Use Cases**:
|
||||||
|
|||||||
@@ -237,7 +237,9 @@ if(YAZE_WITH_GRPC)
|
|||||||
# Add service implementation sources
|
# Add service implementation sources
|
||||||
target_sources(yaze PRIVATE
|
target_sources(yaze PRIVATE
|
||||||
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.cc
|
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.cc
|
||||||
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h)
|
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h
|
||||||
|
${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.cc
|
||||||
|
${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.h)
|
||||||
|
|
||||||
# Link gRPC libraries
|
# Link gRPC libraries
|
||||||
target_link_libraries(yaze PRIVATE
|
target_link_libraries(yaze PRIVATE
|
||||||
|
|||||||
@@ -181,6 +181,12 @@ class ImGuiTestHarnessServiceGrpc final : public ImGuiTestHarness::Service {
|
|||||||
return ConvertStatus(impl_->GetTestResults(request, response));
|
return ConvertStatus(impl_->GetTestResults(request, response));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
grpc::Status DiscoverWidgets(grpc::ServerContext* context,
|
||||||
|
const DiscoverWidgetsRequest* request,
|
||||||
|
DiscoverWidgetsResponse* response) override {
|
||||||
|
return ConvertStatus(impl_->DiscoverWidgets(request, response));
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
static grpc::Status ConvertStatus(const absl::Status& status) {
|
static grpc::Status ConvertStatus(const absl::Status& status) {
|
||||||
if (status.ok()) {
|
if (status.ok()) {
|
||||||
@@ -1146,6 +1152,25 @@ absl::Status ImGuiTestHarnessServiceImpl::GetTestResults(
|
|||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
absl::Status ImGuiTestHarnessServiceImpl::DiscoverWidgets(
|
||||||
|
const DiscoverWidgetsRequest* request,
|
||||||
|
DiscoverWidgetsResponse* response) {
|
||||||
|
if (!request) {
|
||||||
|
return absl::InvalidArgumentError("request cannot be null");
|
||||||
|
}
|
||||||
|
if (!response) {
|
||||||
|
return absl::InvalidArgumentError("response cannot be null");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!test_manager_) {
|
||||||
|
return absl::FailedPreconditionError("TestManager not available");
|
||||||
|
}
|
||||||
|
|
||||||
|
widget_discovery_service_.CollectWidgets(/*ctx=*/nullptr, *request,
|
||||||
|
response);
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// ImGuiTestHarnessServer - Server Lifecycle
|
// ImGuiTestHarnessServer - Server Lifecycle
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|||||||
@@ -8,6 +8,7 @@
|
|||||||
|
|
||||||
#include "absl/status/status.h"
|
#include "absl/status/status.h"
|
||||||
#include "absl/status/statusor.h"
|
#include "absl/status/statusor.h"
|
||||||
|
#include "app/core/widget_discovery_service.h"
|
||||||
|
|
||||||
// Include grpcpp headers for unique_ptr<Server> in member variable
|
// Include grpcpp headers for unique_ptr<Server> in member variable
|
||||||
#include <grpcpp/server.h>
|
#include <grpcpp/server.h>
|
||||||
@@ -42,6 +43,8 @@ class ListTestsRequest;
|
|||||||
class ListTestsResponse;
|
class ListTestsResponse;
|
||||||
class GetTestResultsRequest;
|
class GetTestResultsRequest;
|
||||||
class GetTestResultsResponse;
|
class GetTestResultsResponse;
|
||||||
|
class DiscoverWidgetsRequest;
|
||||||
|
class DiscoverWidgetsResponse;
|
||||||
|
|
||||||
// Implementation of ImGuiTestHarness gRPC service
|
// Implementation of ImGuiTestHarness gRPC service
|
||||||
// This class provides the actual RPC handlers for automated GUI testing
|
// This class provides the actual RPC handlers for automated GUI testing
|
||||||
@@ -85,9 +88,12 @@ class ImGuiTestHarnessServiceImpl {
|
|||||||
ListTestsResponse* response);
|
ListTestsResponse* response);
|
||||||
absl::Status GetTestResults(const GetTestResultsRequest* request,
|
absl::Status GetTestResults(const GetTestResultsRequest* request,
|
||||||
GetTestResultsResponse* response);
|
GetTestResultsResponse* response);
|
||||||
|
absl::Status DiscoverWidgets(const DiscoverWidgetsRequest* request,
|
||||||
|
DiscoverWidgetsResponse* response);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
TestManager* test_manager_; // Non-owning pointer to access ImGuiTestEngine
|
TestManager* test_manager_; // Non-owning pointer to access ImGuiTestEngine
|
||||||
|
WidgetDiscoveryService widget_discovery_service_;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Forward declaration of the gRPC service wrapper
|
// Forward declaration of the gRPC service wrapper
|
||||||
|
|||||||
@@ -27,6 +27,9 @@ service ImGuiTestHarness {
|
|||||||
rpc GetTestStatus(GetTestStatusRequest) returns (GetTestStatusResponse);
|
rpc GetTestStatus(GetTestStatusRequest) returns (GetTestStatusResponse);
|
||||||
rpc ListTests(ListTestsRequest) returns (ListTestsResponse);
|
rpc ListTests(ListTestsRequest) returns (ListTestsResponse);
|
||||||
rpc GetTestResults(GetTestResultsRequest) returns (GetTestResultsResponse);
|
rpc GetTestResults(GetTestResultsRequest) returns (GetTestResultsResponse);
|
||||||
|
|
||||||
|
// Widget discovery API (IT-06)
|
||||||
|
rpc DiscoverWidgets(DiscoverWidgetsRequest) returns (DiscoverWidgetsResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -222,3 +225,60 @@ message AssertionResult {
|
|||||||
string actual_value = 4;
|
string actual_value = 4;
|
||||||
string error_message = 5;
|
string error_message = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// DiscoverWidgets - Enumerate discoverable GUI widgets
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
message DiscoverWidgetsRequest {
|
||||||
|
string window_filter = 1; // Optional: Limit to a window name (case-insensitive)
|
||||||
|
WidgetType type_filter = 2; // Optional: Limit to widget type
|
||||||
|
string path_prefix = 3; // Optional: Require widget path to start with prefix
|
||||||
|
bool include_invisible = 4; // Include widgets that are currently not visible
|
||||||
|
bool include_disabled = 5; // Include widgets that are currently disabled
|
||||||
|
}
|
||||||
|
|
||||||
|
enum WidgetType {
|
||||||
|
WIDGET_TYPE_UNSPECIFIED = 0;
|
||||||
|
WIDGET_TYPE_ALL = 1;
|
||||||
|
WIDGET_TYPE_BUTTON = 2;
|
||||||
|
WIDGET_TYPE_INPUT = 3;
|
||||||
|
WIDGET_TYPE_MENU = 4;
|
||||||
|
WIDGET_TYPE_TAB = 5;
|
||||||
|
WIDGET_TYPE_CHECKBOX = 6;
|
||||||
|
WIDGET_TYPE_SLIDER = 7;
|
||||||
|
WIDGET_TYPE_CANVAS = 8;
|
||||||
|
WIDGET_TYPE_SELECTABLE = 9;
|
||||||
|
WIDGET_TYPE_OTHER = 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
message WidgetBounds {
|
||||||
|
float min_x = 1;
|
||||||
|
float min_y = 2;
|
||||||
|
float max_x = 3;
|
||||||
|
float max_y = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DiscoveredWidget {
|
||||||
|
string path = 1; // Full hierarchical path (e.g. Overworld/Toolset/button:Pan)
|
||||||
|
string label = 2; // Human-readable label (e.g. Pan)
|
||||||
|
string type = 3; // Widget type string (button, input, ...)
|
||||||
|
string description = 4; // Description provided by registry (if any)
|
||||||
|
string suggested_action = 5; // Suggested action for automation (e.g. "Click button:Pan")
|
||||||
|
bool visible = 6; // Currently visible in UI
|
||||||
|
bool enabled = 7; // Currently enabled for interaction
|
||||||
|
WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates
|
||||||
|
uint32 widget_id = 9; // ImGui ID (debugging / direct access)
|
||||||
|
}
|
||||||
|
|
||||||
|
message DiscoveredWindow {
|
||||||
|
string name = 1; // Window name (first segment of path)
|
||||||
|
bool visible = 2; // Whether the window is currently visible
|
||||||
|
repeated DiscoveredWidget widgets = 3; // Widgets contained in this window
|
||||||
|
}
|
||||||
|
|
||||||
|
message DiscoverWidgetsResponse {
|
||||||
|
repeated DiscoveredWindow windows = 1;
|
||||||
|
int32 total_widgets = 2; // Total number of widgets returned
|
||||||
|
int64 generated_at_ms = 3; // Snapshot timestamp (Unix ms)
|
||||||
|
}
|
||||||
|
|||||||
246
src/app/core/widget_discovery_service.cc
Normal file
246
src/app/core/widget_discovery_service.cc
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
#include "app/core/widget_discovery_service.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <map>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
#include "absl/strings/ascii.h"
|
||||||
|
#include "absl/strings/match.h"
|
||||||
|
#include "absl/strings/str_cat.h"
|
||||||
|
#include "absl/time/clock.h"
|
||||||
|
#include "absl/time/time.h"
|
||||||
|
|
||||||
|
namespace yaze {
|
||||||
|
namespace test {
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
struct WindowEntry {
|
||||||
|
int index = -1;
|
||||||
|
bool visible = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
void WidgetDiscoveryService::CollectWidgets(
|
||||||
|
ImGuiTestContext* ctx, const DiscoverWidgetsRequest& request,
|
||||||
|
DiscoverWidgetsResponse* response) const {
|
||||||
|
if (!response) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
response->clear_windows();
|
||||||
|
response->set_total_widgets(0);
|
||||||
|
response->set_generated_at_ms(absl::ToUnixMillis(absl::Now()));
|
||||||
|
|
||||||
|
const auto& registry = gui::WidgetIdRegistry::Instance().GetAllWidgets();
|
||||||
|
if (registry.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string window_filter_lower =
|
||||||
|
absl::AsciiStrToLower(std::string(request.window_filter()));
|
||||||
|
const std::string path_prefix_lower =
|
||||||
|
absl::AsciiStrToLower(std::string(request.path_prefix()));
|
||||||
|
const bool include_invisible = request.include_invisible();
|
||||||
|
const bool include_disabled = request.include_disabled();
|
||||||
|
|
||||||
|
std::map<std::string, WindowEntry> window_lookup;
|
||||||
|
int total_widgets = 0;
|
||||||
|
|
||||||
|
for (const auto& [path, info] : registry) {
|
||||||
|
if (!MatchesType(info.type, request.type_filter())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!MatchesPathPrefix(path, path_prefix_lower)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string window_name = ExtractWindowName(path);
|
||||||
|
if (!MatchesWindow(window_name, window_filter_lower)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
|
||||||
|
WindowEntry& entry = it->second;
|
||||||
|
|
||||||
|
if (inserted) {
|
||||||
|
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||||
|
if (ctx) {
|
||||||
|
ImGuiTestItemInfo window_info =
|
||||||
|
ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError);
|
||||||
|
entry.visible = (window_info.ID != 0);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!include_invisible && !entry.visible) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.index == -1) {
|
||||||
|
DiscoveredWindow* window_proto = response->add_windows();
|
||||||
|
entry.index = response->windows_size() - 1;
|
||||||
|
window_proto->set_name(window_name);
|
||||||
|
window_proto->set_visible(entry.visible);
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string label = ExtractLabel(path);
|
||||||
|
|
||||||
|
bool widget_enabled = true;
|
||||||
|
bool widget_visible = entry.visible;
|
||||||
|
|
||||||
|
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||||
|
bool has_item_info = false;
|
||||||
|
ImGuiTestItemInfo item_info;
|
||||||
|
if (ctx) {
|
||||||
|
item_info = ctx->ItemInfo(label.c_str(), ImGuiTestOpFlags_NoError);
|
||||||
|
if (item_info.ID != 0) {
|
||||||
|
has_item_info = true;
|
||||||
|
widget_visible = item_info.RectClipped.GetWidth() > 0.0f &&
|
||||||
|
item_info.RectClipped.GetHeight() > 0.0f;
|
||||||
|
widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
if (!include_invisible && !widget_visible) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!include_disabled && !widget_enabled) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto* window_proto = response->mutable_windows(entry.index);
|
||||||
|
auto* widget_proto = window_proto->add_widgets();
|
||||||
|
widget_proto->set_path(path);
|
||||||
|
widget_proto->set_label(label);
|
||||||
|
widget_proto->set_type(info.type);
|
||||||
|
widget_proto->set_suggested_action(SuggestedAction(info.type, label));
|
||||||
|
widget_proto->set_visible(widget_visible);
|
||||||
|
widget_proto->set_enabled(widget_enabled);
|
||||||
|
widget_proto->set_widget_id(info.imgui_id);
|
||||||
|
|
||||||
|
if (!info.description.empty()) {
|
||||||
|
widget_proto->set_description(info.description);
|
||||||
|
}
|
||||||
|
|
||||||
|
WidgetBounds* bounds = widget_proto->mutable_bounds();
|
||||||
|
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||||
|
if (ctx && has_item_info) {
|
||||||
|
bounds->set_min_x(item_info.RectFull.Min.x);
|
||||||
|
bounds->set_min_y(item_info.RectFull.Min.y);
|
||||||
|
bounds->set_max_x(item_info.RectFull.Max.x);
|
||||||
|
bounds->set_max_y(item_info.RectFull.Max.y);
|
||||||
|
} else {
|
||||||
|
(void)ctx;
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
(void)ctx;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
++total_widgets;
|
||||||
|
}
|
||||||
|
|
||||||
|
response->set_total_widgets(total_widgets);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool WidgetDiscoveryService::MatchesWindow(absl::string_view window_name,
|
||||||
|
absl::string_view filter_lower) const {
|
||||||
|
if (filter_lower.empty()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
std::string name_lower = absl::AsciiStrToLower(std::string(window_name));
|
||||||
|
return absl::StrContains(name_lower, filter_lower);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool WidgetDiscoveryService::MatchesPathPrefix(absl::string_view path,
|
||||||
|
absl::string_view prefix_lower) const {
|
||||||
|
if (prefix_lower.empty()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
std::string path_lower = absl::AsciiStrToLower(std::string(path));
|
||||||
|
return absl::StartsWith(path_lower, prefix_lower);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool WidgetDiscoveryService::MatchesType(absl::string_view type,
|
||||||
|
WidgetType filter) const {
|
||||||
|
if (filter == WIDGET_TYPE_UNSPECIFIED || filter == WIDGET_TYPE_ALL) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string type_lower = absl::AsciiStrToLower(std::string(type));
|
||||||
|
switch (filter) {
|
||||||
|
case WIDGET_TYPE_BUTTON:
|
||||||
|
return type_lower == "button" || type_lower == "menuitem";
|
||||||
|
case WIDGET_TYPE_INPUT:
|
||||||
|
return type_lower == "input" || type_lower == "textbox" ||
|
||||||
|
type_lower == "inputtext";
|
||||||
|
case WIDGET_TYPE_MENU:
|
||||||
|
return type_lower == "menu" || type_lower == "menuitem";
|
||||||
|
case WIDGET_TYPE_TAB:
|
||||||
|
return type_lower == "tab";
|
||||||
|
case WIDGET_TYPE_CHECKBOX:
|
||||||
|
return type_lower == "checkbox";
|
||||||
|
case WIDGET_TYPE_SLIDER:
|
||||||
|
return type_lower == "slider" || type_lower == "drag";
|
||||||
|
case WIDGET_TYPE_CANVAS:
|
||||||
|
return type_lower == "canvas";
|
||||||
|
case WIDGET_TYPE_SELECTABLE:
|
||||||
|
return type_lower == "selectable";
|
||||||
|
case WIDGET_TYPE_OTHER:
|
||||||
|
return true;
|
||||||
|
default:
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string WidgetDiscoveryService::ExtractWindowName(
|
||||||
|
absl::string_view path) const {
|
||||||
|
size_t slash = path.find('/');
|
||||||
|
if (slash == absl::string_view::npos) {
|
||||||
|
return std::string(path);
|
||||||
|
}
|
||||||
|
return std::string(path.substr(0, slash));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string WidgetDiscoveryService::ExtractLabel(absl::string_view path) const {
|
||||||
|
size_t colon = path.rfind(':');
|
||||||
|
if (colon == absl::string_view::npos) {
|
||||||
|
size_t slash = path.rfind('/');
|
||||||
|
if (slash == absl::string_view::npos) {
|
||||||
|
return std::string(path);
|
||||||
|
}
|
||||||
|
return std::string(path.substr(slash + 1));
|
||||||
|
}
|
||||||
|
return std::string(path.substr(colon + 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string WidgetDiscoveryService::SuggestedAction(absl::string_view type,
|
||||||
|
absl::string_view label) const {
|
||||||
|
std::string type_lower = absl::AsciiStrToLower(std::string(type));
|
||||||
|
if (type_lower == "button" || type_lower == "menuitem") {
|
||||||
|
return absl::StrCat("Click button:", label);
|
||||||
|
}
|
||||||
|
if (type_lower == "checkbox") {
|
||||||
|
return absl::StrCat("Toggle checkbox:", label);
|
||||||
|
}
|
||||||
|
if (type_lower == "slider" || type_lower == "drag") {
|
||||||
|
return absl::StrCat("Adjust slider:", label);
|
||||||
|
}
|
||||||
|
if (type_lower == "input" || type_lower == "textbox" ||
|
||||||
|
type_lower == "inputtext") {
|
||||||
|
return absl::StrCat("Type text into:", label);
|
||||||
|
}
|
||||||
|
if (type_lower == "canvas") {
|
||||||
|
return absl::StrCat("Interact with canvas:", label);
|
||||||
|
}
|
||||||
|
if (type_lower == "tab") {
|
||||||
|
return absl::StrCat("Switch to tab:", label);
|
||||||
|
}
|
||||||
|
return absl::StrCat("Interact with:", label);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace test
|
||||||
|
} // namespace yaze
|
||||||
47
src/app/core/widget_discovery_service.h
Normal file
47
src/app/core/widget_discovery_service.h
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
#ifndef YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
|
||||||
|
#define YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "absl/strings/string_view.h"
|
||||||
|
#include "app/core/proto/imgui_test_harness.pb.h"
|
||||||
|
#include "app/gui/widget_id_registry.h"
|
||||||
|
|
||||||
|
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||||
|
#include "imgui_test_engine/imgui_te_context.h"
|
||||||
|
#else
|
||||||
|
struct ImGuiTestContext;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace yaze {
|
||||||
|
namespace test {
|
||||||
|
|
||||||
|
// Service responsible for transforming widget registry data into
|
||||||
|
// DiscoverWidgetsResponse payloads.
|
||||||
|
class WidgetDiscoveryService {
|
||||||
|
public:
|
||||||
|
WidgetDiscoveryService() = default;
|
||||||
|
|
||||||
|
void CollectWidgets(ImGuiTestContext* ctx,
|
||||||
|
const DiscoverWidgetsRequest& request,
|
||||||
|
DiscoverWidgetsResponse* response) const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool MatchesWindow(absl::string_view window_name,
|
||||||
|
absl::string_view filter) const;
|
||||||
|
bool MatchesPathPrefix(absl::string_view path,
|
||||||
|
absl::string_view prefix) const;
|
||||||
|
bool MatchesType(absl::string_view type,
|
||||||
|
WidgetType filter) const;
|
||||||
|
|
||||||
|
std::string ExtractWindowName(absl::string_view path) const;
|
||||||
|
std::string ExtractLabel(absl::string_view path) const;
|
||||||
|
std::string SuggestedAction(absl::string_view type,
|
||||||
|
absl::string_view label) const;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace test
|
||||||
|
} // namespace yaze
|
||||||
|
|
||||||
|
#endif // YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
|
||||||
@@ -25,6 +25,8 @@
|
|||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <limits>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
// Declare the rom flag so we can access it
|
// Declare the rom flag so we can access it
|
||||||
ABSL_DECLARE_FLAG(std::string, rom);
|
ABSL_DECLARE_FLAG(std::string, rom);
|
||||||
@@ -105,6 +107,45 @@ std::optional<TestRunStatus> ParseStatusFilter(absl::string_view value) {
|
|||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::optional<WidgetTypeFilter> ParseWidgetTypeFilter(
|
||||||
|
absl::string_view value) {
|
||||||
|
std::string lower = std::string(absl::AsciiStrToLower(value));
|
||||||
|
if (lower.empty() || lower == "unspecified" || lower == "any") {
|
||||||
|
return WidgetTypeFilter::kUnspecified;
|
||||||
|
}
|
||||||
|
if (lower == "all") {
|
||||||
|
return WidgetTypeFilter::kAll;
|
||||||
|
}
|
||||||
|
if (lower == "button" || lower == "buttons") {
|
||||||
|
return WidgetTypeFilter::kButton;
|
||||||
|
}
|
||||||
|
if (lower == "input" || lower == "textbox" || lower == "field") {
|
||||||
|
return WidgetTypeFilter::kInput;
|
||||||
|
}
|
||||||
|
if (lower == "menu" || lower == "menuitem" || lower == "menu-item") {
|
||||||
|
return WidgetTypeFilter::kMenu;
|
||||||
|
}
|
||||||
|
if (lower == "tab" || lower == "tabs") {
|
||||||
|
return WidgetTypeFilter::kTab;
|
||||||
|
}
|
||||||
|
if (lower == "checkbox" || lower == "toggle") {
|
||||||
|
return WidgetTypeFilter::kCheckbox;
|
||||||
|
}
|
||||||
|
if (lower == "slider" || lower == "drag" || lower == "sliderfloat") {
|
||||||
|
return WidgetTypeFilter::kSlider;
|
||||||
|
}
|
||||||
|
if (lower == "canvas" || lower == "viewport") {
|
||||||
|
return WidgetTypeFilter::kCanvas;
|
||||||
|
}
|
||||||
|
if (lower == "selectable" || lower == "list-item") {
|
||||||
|
return WidgetTypeFilter::kSelectable;
|
||||||
|
}
|
||||||
|
if (lower == "other") {
|
||||||
|
return WidgetTypeFilter::kOther;
|
||||||
|
}
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
std::string HarnessAddress(const std::string& host, int port) {
|
std::string HarnessAddress(const std::string& host, int port) {
|
||||||
return absl::StrFormat("%s:%d", host, port);
|
return absl::StrFormat("%s:%d", host, port);
|
||||||
}
|
}
|
||||||
@@ -1051,6 +1092,327 @@ absl::Status HandleTestCommand(const std::vector<std::string>& arg_vec) {
|
|||||||
return HandleTestRunCommand(arg_vec);
|
return HandleTestRunCommand(arg_vec);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
|
||||||
|
std::string host = "localhost";
|
||||||
|
int port = 50052;
|
||||||
|
std::string window_filter;
|
||||||
|
std::string path_prefix;
|
||||||
|
std::optional<WidgetTypeFilter> type_filter;
|
||||||
|
std::optional<std::string> type_filter_label;
|
||||||
|
bool include_invisible = false;
|
||||||
|
bool include_disabled = false;
|
||||||
|
std::string format = "table";
|
||||||
|
int limit = -1;
|
||||||
|
|
||||||
|
auto require_value = [&](const std::vector<std::string>& args, size_t& index,
|
||||||
|
absl::string_view flag) -> absl::StatusOr<std::string> {
|
||||||
|
if (index + 1 >= args.size()) {
|
||||||
|
return absl::InvalidArgumentError(
|
||||||
|
absl::StrFormat("Flag %s requires a value", flag));
|
||||||
|
}
|
||||||
|
return args[++index];
|
||||||
|
};
|
||||||
|
|
||||||
|
for (size_t i = 0; i < arg_vec.size(); ++i) {
|
||||||
|
const std::string& token = arg_vec[i];
|
||||||
|
if (token == "--host") {
|
||||||
|
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--host"));
|
||||||
|
host = std::move(value);
|
||||||
|
} else if (absl::StartsWith(token, "--host=")) {
|
||||||
|
host = token.substr(7);
|
||||||
|
} else if (token == "--port") {
|
||||||
|
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--port"));
|
||||||
|
port = std::stoi(value);
|
||||||
|
} else if (absl::StartsWith(token, "--port=")) {
|
||||||
|
port = std::stoi(token.substr(7));
|
||||||
|
} else if (token == "--window" || token == "--window-filter") {
|
||||||
|
ASSIGN_OR_RETURN(auto value,
|
||||||
|
require_value(arg_vec, i, token.c_str()));
|
||||||
|
window_filter = std::move(value);
|
||||||
|
} else if (absl::StartsWith(token, "--window=")) {
|
||||||
|
window_filter = token.substr(9);
|
||||||
|
} else if (token == "--path-prefix") {
|
||||||
|
ASSIGN_OR_RETURN(auto value,
|
||||||
|
require_value(arg_vec, i, "--path-prefix"));
|
||||||
|
path_prefix = std::move(value);
|
||||||
|
} else if (absl::StartsWith(token, "--path-prefix=")) {
|
||||||
|
path_prefix = token.substr(14);
|
||||||
|
} else if (token == "--type") {
|
||||||
|
ASSIGN_OR_RETURN(auto value,
|
||||||
|
require_value(arg_vec, i, "--type"));
|
||||||
|
auto parsed = ParseWidgetTypeFilter(value);
|
||||||
|
if (!parsed.has_value()) {
|
||||||
|
return absl::InvalidArgumentError(
|
||||||
|
absl::StrFormat("Unknown widget type filter: %s", value));
|
||||||
|
}
|
||||||
|
type_filter = parsed;
|
||||||
|
type_filter_label = absl::AsciiStrToLower(value);
|
||||||
|
} else if (absl::StartsWith(token, "--type=")) {
|
||||||
|
std::string value = token.substr(7);
|
||||||
|
auto parsed = ParseWidgetTypeFilter(value);
|
||||||
|
if (!parsed.has_value()) {
|
||||||
|
return absl::InvalidArgumentError(
|
||||||
|
absl::StrFormat("Unknown widget type filter: %s", value));
|
||||||
|
}
|
||||||
|
type_filter = parsed;
|
||||||
|
type_filter_label = absl::AsciiStrToLower(value);
|
||||||
|
} else if (token == "--include-invisible") {
|
||||||
|
include_invisible = true;
|
||||||
|
} else if (token == "--include-disabled") {
|
||||||
|
include_disabled = true;
|
||||||
|
} else if (token == "--format") {
|
||||||
|
ASSIGN_OR_RETURN(auto value,
|
||||||
|
require_value(arg_vec, i, "--format"));
|
||||||
|
format = std::move(value);
|
||||||
|
} else if (absl::StartsWith(token, "--format=")) {
|
||||||
|
format = token.substr(9);
|
||||||
|
} else if (token == "--limit") {
|
||||||
|
ASSIGN_OR_RETURN(auto value,
|
||||||
|
require_value(arg_vec, i, "--limit"));
|
||||||
|
limit = std::stoi(value);
|
||||||
|
} else if (absl::StartsWith(token, "--limit=")) {
|
||||||
|
limit = std::stoi(token.substr(8));
|
||||||
|
} else if (token == "--help" || token == "-h") {
|
||||||
|
std::cout << "Usage: agent gui discover [options]\n"
|
||||||
|
<< " --host <host>\n"
|
||||||
|
<< " --port <port>\n"
|
||||||
|
<< " --window <name>\n"
|
||||||
|
<< " --type <widget-type>\n"
|
||||||
|
<< " --path-prefix <path>\n"
|
||||||
|
<< " --include-invisible\n"
|
||||||
|
<< " --include-disabled\n"
|
||||||
|
<< " --format <table|json>\n"
|
||||||
|
<< " --limit <n>\n";
|
||||||
|
return absl::OkStatus();
|
||||||
|
} else {
|
||||||
|
return absl::InvalidArgumentError(
|
||||||
|
absl::StrFormat("Unknown flag for agent gui discover: %s",
|
||||||
|
token));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
format = absl::AsciiStrToLower(format);
|
||||||
|
if (format != "table" && format != "json") {
|
||||||
|
return absl::InvalidArgumentError(
|
||||||
|
"--format must be either 'table' or 'json'");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (limit == 0) {
|
||||||
|
return absl::InvalidArgumentError("--limit must be positive");
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifndef YAZE_WITH_GRPC
|
||||||
|
(void)host;
|
||||||
|
(void)port;
|
||||||
|
(void)window_filter;
|
||||||
|
(void)path_prefix;
|
||||||
|
(void)type_filter;
|
||||||
|
(void)include_invisible;
|
||||||
|
(void)include_disabled;
|
||||||
|
(void)format;
|
||||||
|
(void)limit;
|
||||||
|
return absl::UnimplementedError(
|
||||||
|
"GUI automation requires YAZE_WITH_GRPC=ON at build time.\n"
|
||||||
|
"Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON");
|
||||||
|
#else
|
||||||
|
GuiAutomationClient client(HarnessAddress(host, port));
|
||||||
|
RETURN_IF_ERROR(client.Connect());
|
||||||
|
|
||||||
|
DiscoverWidgetsQuery query;
|
||||||
|
query.window_filter = window_filter;
|
||||||
|
query.path_prefix = path_prefix;
|
||||||
|
if (type_filter.has_value()) {
|
||||||
|
query.type_filter = type_filter.value();
|
||||||
|
}
|
||||||
|
query.include_invisible = include_invisible;
|
||||||
|
query.include_disabled = include_disabled;
|
||||||
|
|
||||||
|
ASSIGN_OR_RETURN(auto response, client.DiscoverWidgets(query));
|
||||||
|
|
||||||
|
int max_items = limit > 0 ? limit : std::numeric_limits<int>::max();
|
||||||
|
int remaining = max_items;
|
||||||
|
std::vector<DiscoveredWindowInfo> trimmed_windows;
|
||||||
|
trimmed_windows.reserve(response.windows.size());
|
||||||
|
int rendered_widgets = 0;
|
||||||
|
|
||||||
|
for (const auto& window : response.windows) {
|
||||||
|
if (remaining <= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
DiscoveredWindowInfo trimmed;
|
||||||
|
trimmed.name = window.name;
|
||||||
|
trimmed.visible = window.visible;
|
||||||
|
|
||||||
|
for (const auto& widget : window.widgets) {
|
||||||
|
if (remaining <= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
trimmed.widgets.push_back(widget);
|
||||||
|
--remaining;
|
||||||
|
++rendered_widgets;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!trimmed.widgets.empty()) {
|
||||||
|
trimmed_windows.push_back(std::move(trimmed));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool truncated = rendered_widgets < response.total_widgets;
|
||||||
|
|
||||||
|
if (format == "json") {
|
||||||
|
std::cout << "{\n";
|
||||||
|
std::cout << " \"server\": \""
|
||||||
|
<< JsonEscape(HarnessAddress(host, port)) << "\",\n";
|
||||||
|
std::cout << " \"totalWidgets\": " << response.total_widgets << ",\n";
|
||||||
|
std::cout << " \"returnedWidgets\": " << rendered_widgets << ",\n";
|
||||||
|
std::cout << " \"truncated\": " << (truncated ? "true" : "false")
|
||||||
|
<< ",\n";
|
||||||
|
std::cout << " \"generatedAt\": "
|
||||||
|
<< (response.generated_at.has_value()
|
||||||
|
? absl::StrCat("\"",
|
||||||
|
JsonEscape(absl::FormatTime(
|
||||||
|
"%Y-%m-%dT%H:%M:%SZ",
|
||||||
|
*response.generated_at,
|
||||||
|
absl::UTCTimeZone())),
|
||||||
|
"\"")
|
||||||
|
: std::string("null"))
|
||||||
|
<< ",\n";
|
||||||
|
std::cout << " \"windows\": [\n";
|
||||||
|
|
||||||
|
for (size_t w = 0; w < trimmed_windows.size(); ++w) {
|
||||||
|
const auto& window = trimmed_windows[w];
|
||||||
|
std::cout << " {\n";
|
||||||
|
std::cout << " \"name\": \"" << JsonEscape(window.name)
|
||||||
|
<< "\",\n";
|
||||||
|
std::cout << " \"visible\": "
|
||||||
|
<< (window.visible ? "true" : "false") << ",\n";
|
||||||
|
std::cout << " \"widgets\": [\n";
|
||||||
|
for (size_t i = 0; i < window.widgets.size(); ++i) {
|
||||||
|
const auto& widget = window.widgets[i];
|
||||||
|
std::cout << " {\n";
|
||||||
|
std::cout << " \"path\": \""
|
||||||
|
<< JsonEscape(widget.path) << "\",\n";
|
||||||
|
std::cout << " \"label\": \""
|
||||||
|
<< JsonEscape(widget.label) << "\",\n";
|
||||||
|
std::cout << " \"type\": \""
|
||||||
|
<< JsonEscape(widget.type) << "\",\n";
|
||||||
|
std::cout << " \"description\": \""
|
||||||
|
<< JsonEscape(widget.description) << "\",\n";
|
||||||
|
std::cout << " \"suggestedAction\": \""
|
||||||
|
<< JsonEscape(widget.suggested_action) << "\",\n";
|
||||||
|
std::cout << " \"visible\": "
|
||||||
|
<< (widget.visible ? "true" : "false") << ",\n";
|
||||||
|
std::cout << " \"enabled\": "
|
||||||
|
<< (widget.enabled ? "true" : "false") << ",\n";
|
||||||
|
std::cout << " \"bounds\": { \"min\": ["
|
||||||
|
<< widget.bounds.min_x << ", "
|
||||||
|
<< widget.bounds.min_y << "], \"max\": ["
|
||||||
|
<< widget.bounds.max_x << ", "
|
||||||
|
<< widget.bounds.max_y << "] },\n";
|
||||||
|
std::cout << " \"widgetId\": " << widget.widget_id
|
||||||
|
<< "\n";
|
||||||
|
std::cout << " }";
|
||||||
|
if (i + 1 < window.widgets.size()) {
|
||||||
|
std::cout << ",";
|
||||||
|
}
|
||||||
|
std::cout << "\n";
|
||||||
|
}
|
||||||
|
std::cout << " ]\n";
|
||||||
|
std::cout << " }";
|
||||||
|
if (w + 1 < trimmed_windows.size()) {
|
||||||
|
std::cout << ",";
|
||||||
|
}
|
||||||
|
std::cout << "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << " ]\n";
|
||||||
|
std::cout << "}\n";
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "\n=== Widget Discovery ===\n";
|
||||||
|
std::cout << "Server: " << HarnessAddress(host, port) << "\n";
|
||||||
|
if (!window_filter.empty()) {
|
||||||
|
std::cout << "Window filter: " << window_filter << "\n";
|
||||||
|
}
|
||||||
|
if (!path_prefix.empty()) {
|
||||||
|
std::cout << "Path prefix: " << path_prefix << "\n";
|
||||||
|
}
|
||||||
|
if (type_filter_label.has_value()) {
|
||||||
|
std::cout << "Type filter: " << *type_filter_label << "\n";
|
||||||
|
}
|
||||||
|
std::cout << "Include invisible: " << (include_invisible ? "yes" : "no")
|
||||||
|
<< "\n";
|
||||||
|
std::cout << "Include disabled: " << (include_disabled ? "yes" : "no")
|
||||||
|
<< "\n\n";
|
||||||
|
|
||||||
|
if (trimmed_windows.empty()) {
|
||||||
|
std::cout << "No widgets matched the provided filters." << std::endl;
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const auto& window : trimmed_windows) {
|
||||||
|
std::cout << "Window: " << window.name
|
||||||
|
<< (window.visible ? " (visible)" : " (hidden)")
|
||||||
|
<< "\n";
|
||||||
|
for (const auto& widget : window.widgets) {
|
||||||
|
std::cout << " • [" << widget.type << "] " << widget.label
|
||||||
|
<< "\n";
|
||||||
|
std::cout << " Path: " << widget.path << "\n";
|
||||||
|
if (!widget.description.empty()) {
|
||||||
|
std::cout << " Description: " << widget.description
|
||||||
|
<< "\n";
|
||||||
|
}
|
||||||
|
std::cout << " Suggested: " << widget.suggested_action << "\n";
|
||||||
|
std::cout << " State: "
|
||||||
|
<< (widget.visible ? "visible" : "hidden") << ", "
|
||||||
|
<< (widget.enabled ? "enabled" : "disabled") << "\n";
|
||||||
|
std::cout << absl::StrFormat(
|
||||||
|
" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n",
|
||||||
|
widget.bounds.min_x, widget.bounds.min_y,
|
||||||
|
widget.bounds.max_x, widget.bounds.max_y);
|
||||||
|
std::cout << " Widget ID: 0x" << std::hex << widget.widget_id
|
||||||
|
<< std::dec << "\n";
|
||||||
|
}
|
||||||
|
std::cout << "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Widgets shown: " << rendered_widgets << " of "
|
||||||
|
<< response.total_widgets;
|
||||||
|
if (truncated) {
|
||||||
|
std::cout << " (truncated)";
|
||||||
|
}
|
||||||
|
std::cout << "\n";
|
||||||
|
|
||||||
|
if (response.generated_at.has_value()) {
|
||||||
|
std::cout << "Snapshot: "
|
||||||
|
<< absl::FormatTime("%Y-%m-%d %H:%M:%S",
|
||||||
|
*response.generated_at,
|
||||||
|
absl::LocalTimeZone())
|
||||||
|
<< "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status HandleGuiCommand(const std::vector<std::string>& arg_vec) {
|
||||||
|
if (arg_vec.empty()) {
|
||||||
|
return absl::InvalidArgumentError(
|
||||||
|
"Usage: agent gui <discover> [options]");
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string& subcommand = arg_vec[0];
|
||||||
|
std::vector<std::string> tail(arg_vec.begin() + 1, arg_vec.end());
|
||||||
|
|
||||||
|
if (subcommand == "discover") {
|
||||||
|
return HandleGuiDiscoverCommand(tail);
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::InvalidArgumentError(
|
||||||
|
absl::StrFormat("Unknown agent gui subcommand: %s", subcommand));
|
||||||
|
}
|
||||||
|
|
||||||
absl::Status HandleLearnCommand() {
|
absl::Status HandleLearnCommand() {
|
||||||
std::cout << "Agent learn not yet implemented." << std::endl;
|
std::cout << "Agent learn not yet implemented." << std::endl;
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
@@ -1184,7 +1546,7 @@ absl::Status HandleDescribeCommand(const std::vector<std::string>& arg_vec) {
|
|||||||
absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
|
absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
|
||||||
if (arg_vec.empty()) {
|
if (arg_vec.empty()) {
|
||||||
return absl::InvalidArgumentError(
|
return absl::InvalidArgumentError(
|
||||||
"Usage: agent <run|plan|diff|test|learn|list|commit|revert|describe> [options]");
|
"Usage: agent <run|plan|diff|test|gui|learn|list|commit|revert|describe> [options]");
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string subcommand = arg_vec[0];
|
std::string subcommand = arg_vec[0];
|
||||||
@@ -1198,6 +1560,8 @@ absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
|
|||||||
return HandleDiffCommand(rom_, subcommand_args);
|
return HandleDiffCommand(rom_, subcommand_args);
|
||||||
} else if (subcommand == "test") {
|
} else if (subcommand == "test") {
|
||||||
return HandleTestCommand(subcommand_args);
|
return HandleTestCommand(subcommand_args);
|
||||||
|
} else if (subcommand == "gui") {
|
||||||
|
return HandleGuiCommand(subcommand_args);
|
||||||
} else if (subcommand == "learn") {
|
} else if (subcommand == "learn") {
|
||||||
return HandleLearnCommand();
|
return HandleLearnCommand();
|
||||||
} else if (subcommand == "list") {
|
} else if (subcommand == "list") {
|
||||||
|
|||||||
@@ -60,11 +60,13 @@ void ModernCLI::SetupCommands() {
|
|||||||
commands_["agent"] = {
|
commands_["agent"] = {
|
||||||
.name = "agent",
|
.name = "agent",
|
||||||
.description = "Interact with the AI agent",
|
.description = "Interact with the AI agent",
|
||||||
.usage = "z3ed agent <run|plan|diff|test|list|learn|commit|revert|describe> [options]\n"
|
.usage = "z3ed agent <run|plan|diff|test|gui|list|learn|commit|revert|describe> [options]\n"
|
||||||
" test run: --prompt \"<description>\" [--host <host>] [--port <port>] [--timeout <sec>]\n"
|
" test run: --prompt \"<description>\" [--host <host>] [--port <port>] [--timeout <sec>]\n"
|
||||||
" test status: status --test-id <id> [--follow] [--host <host>] [--port <port>]\n"
|
" test status: status --test-id <id> [--follow] [--host <host>] [--port <port>]\n"
|
||||||
" test list: list [--category <name>] [--status <state>] [--limit <n>] [--host <host>] [--port <port>]\n"
|
" test list: list [--category <name>] [--status <state>] [--limit <n>] [--host <host>] [--port <port>]\n"
|
||||||
" test results: results --test-id <id> [--include-logs] [--format yaml|json] [--host <host>] [--port <port>]\n"
|
" test results: results --test-id <id> [--include-logs] [--format yaml|json] [--host <host>] [--port <port>]\n"
|
||||||
|
" gui discover: discover [--window <name>] [--type <widget>] [--path-prefix <path>]\n"
|
||||||
|
" [--include-invisible] [--include-disabled] [--format table|json] [--limit <n>]\n"
|
||||||
" describe options: [--resource <name>] [--format json|yaml] [--output <path>]\n"
|
" describe options: [--resource <name>] [--format json|yaml] [--output <path>]\n"
|
||||||
" [--version <value>] [--last-updated <date>]",
|
" [--version <value>] [--last-updated <date>]",
|
||||||
.handler = [this](const std::vector<std::string>& args) -> absl::Status {
|
.handler = [this](const std::vector<std::string>& args) -> absl::Status {
|
||||||
|
|||||||
@@ -21,6 +21,35 @@ std::optional<absl::Time> OptionalTimeFromMillis(int64_t millis) {
|
|||||||
return absl::FromUnixMillis(millis);
|
return absl::FromUnixMillis(millis);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
yaze::test::WidgetType ConvertWidgetTypeFilterToProto(WidgetTypeFilter filter) {
|
||||||
|
using ProtoType = yaze::test::WidgetType;
|
||||||
|
switch (filter) {
|
||||||
|
case WidgetTypeFilter::kAll:
|
||||||
|
return ProtoType::WIDGET_TYPE_ALL;
|
||||||
|
case WidgetTypeFilter::kButton:
|
||||||
|
return ProtoType::WIDGET_TYPE_BUTTON;
|
||||||
|
case WidgetTypeFilter::kInput:
|
||||||
|
return ProtoType::WIDGET_TYPE_INPUT;
|
||||||
|
case WidgetTypeFilter::kMenu:
|
||||||
|
return ProtoType::WIDGET_TYPE_MENU;
|
||||||
|
case WidgetTypeFilter::kTab:
|
||||||
|
return ProtoType::WIDGET_TYPE_TAB;
|
||||||
|
case WidgetTypeFilter::kCheckbox:
|
||||||
|
return ProtoType::WIDGET_TYPE_CHECKBOX;
|
||||||
|
case WidgetTypeFilter::kSlider:
|
||||||
|
return ProtoType::WIDGET_TYPE_SLIDER;
|
||||||
|
case WidgetTypeFilter::kCanvas:
|
||||||
|
return ProtoType::WIDGET_TYPE_CANVAS;
|
||||||
|
case WidgetTypeFilter::kSelectable:
|
||||||
|
return ProtoType::WIDGET_TYPE_SELECTABLE;
|
||||||
|
case WidgetTypeFilter::kOther:
|
||||||
|
return ProtoType::WIDGET_TYPE_OTHER;
|
||||||
|
case WidgetTypeFilter::kUnspecified:
|
||||||
|
default:
|
||||||
|
return ProtoType::WIDGET_TYPE_UNSPECIFIED;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
TestRunStatus ConvertStatusProto(
|
TestRunStatus ConvertStatusProto(
|
||||||
yaze::test::GetTestStatusResponse::Status status) {
|
yaze::test::GetTestStatusResponse::Status status) {
|
||||||
using ProtoStatus = yaze::test::GetTestStatusResponse::Status;
|
using ProtoStatus = yaze::test::GetTestStatusResponse::Status;
|
||||||
@@ -440,5 +469,73 @@ absl::StatusOr<TestResultDetails> GuiAutomationClient::GetTestResults(
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
absl::StatusOr<DiscoverWidgetsResult> GuiAutomationClient::DiscoverWidgets(
|
||||||
|
const DiscoverWidgetsQuery& query) {
|
||||||
|
#ifdef YAZE_WITH_GRPC
|
||||||
|
if (!stub_) {
|
||||||
|
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||||
|
}
|
||||||
|
|
||||||
|
yaze::test::DiscoverWidgetsRequest request;
|
||||||
|
if (!query.window_filter.empty()) {
|
||||||
|
request.set_window_filter(query.window_filter);
|
||||||
|
}
|
||||||
|
request.set_type_filter(ConvertWidgetTypeFilterToProto(query.type_filter));
|
||||||
|
if (!query.path_prefix.empty()) {
|
||||||
|
request.set_path_prefix(query.path_prefix);
|
||||||
|
}
|
||||||
|
request.set_include_invisible(query.include_invisible);
|
||||||
|
request.set_include_disabled(query.include_disabled);
|
||||||
|
|
||||||
|
yaze::test::DiscoverWidgetsResponse response;
|
||||||
|
grpc::ClientContext context;
|
||||||
|
grpc::Status status = stub_->DiscoverWidgets(&context, request, &response);
|
||||||
|
|
||||||
|
if (!status.ok()) {
|
||||||
|
return absl::InternalError(
|
||||||
|
absl::StrFormat("DiscoverWidgets RPC failed: %s",
|
||||||
|
status.error_message()));
|
||||||
|
}
|
||||||
|
|
||||||
|
DiscoverWidgetsResult result;
|
||||||
|
result.total_widgets = response.total_widgets();
|
||||||
|
if (response.generated_at_ms() > 0) {
|
||||||
|
result.generated_at = OptionalTimeFromMillis(response.generated_at_ms());
|
||||||
|
}
|
||||||
|
|
||||||
|
result.windows.reserve(response.windows_size());
|
||||||
|
for (const auto& window_proto : response.windows()) {
|
||||||
|
DiscoveredWindowInfo window_info;
|
||||||
|
window_info.name = window_proto.name();
|
||||||
|
window_info.visible = window_proto.visible();
|
||||||
|
window_info.widgets.reserve(window_proto.widgets_size());
|
||||||
|
|
||||||
|
for (const auto& widget_proto : window_proto.widgets()) {
|
||||||
|
WidgetDescriptor widget;
|
||||||
|
widget.path = widget_proto.path();
|
||||||
|
widget.label = widget_proto.label();
|
||||||
|
widget.type = widget_proto.type();
|
||||||
|
widget.description = widget_proto.description();
|
||||||
|
widget.suggested_action = widget_proto.suggested_action();
|
||||||
|
widget.visible = widget_proto.visible();
|
||||||
|
widget.enabled = widget_proto.enabled();
|
||||||
|
widget.bounds.min_x = widget_proto.bounds().min_x();
|
||||||
|
widget.bounds.min_y = widget_proto.bounds().min_y();
|
||||||
|
widget.bounds.max_x = widget_proto.bounds().max_x();
|
||||||
|
widget.bounds.max_y = widget_proto.bounds().max_y();
|
||||||
|
widget.widget_id = widget_proto.widget_id();
|
||||||
|
window_info.widgets.push_back(std::move(widget));
|
||||||
|
}
|
||||||
|
|
||||||
|
result.windows.push_back(std::move(window_info));
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
#else
|
||||||
|
(void)query;
|
||||||
|
return absl::UnimplementedError("gRPC not available");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace cli
|
} // namespace cli
|
||||||
} // namespace yaze
|
} // namespace yaze
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
#include "absl/time/time.h"
|
#include "absl/time/time.h"
|
||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
#include <cstdint>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
@@ -120,6 +121,59 @@ struct TestResultDetails {
|
|||||||
std::map<std::string, int> metrics;
|
std::map<std::string, int> metrics;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum class WidgetTypeFilter {
|
||||||
|
kUnspecified,
|
||||||
|
kAll,
|
||||||
|
kButton,
|
||||||
|
kInput,
|
||||||
|
kMenu,
|
||||||
|
kTab,
|
||||||
|
kCheckbox,
|
||||||
|
kSlider,
|
||||||
|
kCanvas,
|
||||||
|
kSelectable,
|
||||||
|
kOther,
|
||||||
|
};
|
||||||
|
|
||||||
|
struct WidgetBoundingBox {
|
||||||
|
float min_x = 0.0f;
|
||||||
|
float min_y = 0.0f;
|
||||||
|
float max_x = 0.0f;
|
||||||
|
float max_y = 0.0f;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct WidgetDescriptor {
|
||||||
|
std::string path;
|
||||||
|
std::string label;
|
||||||
|
std::string type;
|
||||||
|
std::string description;
|
||||||
|
std::string suggested_action;
|
||||||
|
bool visible = true;
|
||||||
|
bool enabled = true;
|
||||||
|
WidgetBoundingBox bounds;
|
||||||
|
uint32_t widget_id = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct DiscoveredWindowInfo {
|
||||||
|
std::string name;
|
||||||
|
bool visible = true;
|
||||||
|
std::vector<WidgetDescriptor> widgets;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct DiscoverWidgetsQuery {
|
||||||
|
std::string window_filter;
|
||||||
|
WidgetTypeFilter type_filter = WidgetTypeFilter::kUnspecified;
|
||||||
|
std::string path_prefix;
|
||||||
|
bool include_invisible = false;
|
||||||
|
bool include_disabled = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct DiscoverWidgetsResult {
|
||||||
|
std::vector<DiscoveredWindowInfo> windows;
|
||||||
|
int total_widgets = 0;
|
||||||
|
std::optional<absl::Time> generated_at;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Client for automating YAZE GUI through gRPC
|
* @brief Client for automating YAZE GUI through gRPC
|
||||||
*
|
*
|
||||||
@@ -225,6 +279,9 @@ class GuiAutomationClient {
|
|||||||
absl::StatusOr<TestResultDetails> GetTestResults(const std::string& test_id,
|
absl::StatusOr<TestResultDetails> GetTestResults(const std::string& test_id,
|
||||||
bool include_logs = false);
|
bool include_logs = false);
|
||||||
|
|
||||||
|
absl::StatusOr<DiscoverWidgetsResult> DiscoverWidgets(
|
||||||
|
const DiscoverWidgetsQuery& query);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Check if client is connected
|
* @brief Check if client is connected
|
||||||
*/
|
*/
|
||||||
|
|||||||
Reference in New Issue
Block a user