feat: Implement widget discovery feature in GUI automation
- Added `DiscoverWidgets` RPC to the ImGuiTestHarness service for enumerating GUI widgets. - Introduced `WidgetDiscoveryService` to handle widget collection and filtering based on various criteria. - Updated `agent gui discover` command to support new options for filtering and output formats. - Enhanced `GuiAutomationClient` to facilitate widget discovery requests and responses. - Added necessary protobuf messages for widget discovery in `imgui_test_harness.proto`. - Updated CLI command handling to include new GUI discovery functionality. - Improved documentation for the `agent gui discover` command with examples and output formats.
This commit is contained in:
scawful
@@ -237,7 +237,9 @@ if(YAZE_WITH_GRPC)
|
||||
# Add service implementation sources
|
||||
target_sources(yaze PRIVATE
|
||||
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.cc
|
||||
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h)
|
||||
${CMAKE_SOURCE_DIR}/src/app/core/imgui_test_harness_service.h
|
||||
${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.cc
|
||||
${CMAKE_SOURCE_DIR}/src/app/core/widget_discovery_service.h)
|
||||
|
||||
# Link gRPC libraries
|
||||
target_link_libraries(yaze PRIVATE
|
||||
|
||||
@@ -181,6 +181,12 @@ class ImGuiTestHarnessServiceGrpc final : public ImGuiTestHarness::Service {
|
||||
return ConvertStatus(impl_->GetTestResults(request, response));
|
||||
}
|
||||
|
||||
grpc::Status DiscoverWidgets(grpc::ServerContext* context,
|
||||
const DiscoverWidgetsRequest* request,
|
||||
DiscoverWidgetsResponse* response) override {
|
||||
return ConvertStatus(impl_->DiscoverWidgets(request, response));
|
||||
}
|
||||
|
||||
private:
|
||||
static grpc::Status ConvertStatus(const absl::Status& status) {
|
||||
if (status.ok()) {
|
||||
@@ -1146,6 +1152,25 @@ absl::Status ImGuiTestHarnessServiceImpl::GetTestResults(
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status ImGuiTestHarnessServiceImpl::DiscoverWidgets(
|
||||
const DiscoverWidgetsRequest* request,
|
||||
DiscoverWidgetsResponse* response) {
|
||||
if (!request) {
|
||||
return absl::InvalidArgumentError("request cannot be null");
|
||||
}
|
||||
if (!response) {
|
||||
return absl::InvalidArgumentError("response cannot be null");
|
||||
}
|
||||
|
||||
if (!test_manager_) {
|
||||
return absl::FailedPreconditionError("TestManager not available");
|
||||
}
|
||||
|
||||
widget_discovery_service_.CollectWidgets(/*ctx=*/nullptr, *request,
|
||||
response);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ImGuiTestHarnessServer - Server Lifecycle
|
||||
// ============================================================================
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "app/core/widget_discovery_service.h"
|
||||
|
||||
// Include grpcpp headers for unique_ptr<Server> in member variable
|
||||
#include <grpcpp/server.h>
|
||||
@@ -42,6 +43,8 @@ class ListTestsRequest;
|
||||
class ListTestsResponse;
|
||||
class GetTestResultsRequest;
|
||||
class GetTestResultsResponse;
|
||||
class DiscoverWidgetsRequest;
|
||||
class DiscoverWidgetsResponse;
|
||||
|
||||
// Implementation of ImGuiTestHarness gRPC service
|
||||
// This class provides the actual RPC handlers for automated GUI testing
|
||||
@@ -85,9 +88,12 @@ class ImGuiTestHarnessServiceImpl {
|
||||
ListTestsResponse* response);
|
||||
absl::Status GetTestResults(const GetTestResultsRequest* request,
|
||||
GetTestResultsResponse* response);
|
||||
absl::Status DiscoverWidgets(const DiscoverWidgetsRequest* request,
|
||||
DiscoverWidgetsResponse* response);
|
||||
|
||||
private:
|
||||
TestManager* test_manager_; // Non-owning pointer to access ImGuiTestEngine
|
||||
WidgetDiscoveryService widget_discovery_service_;
|
||||
};
|
||||
|
||||
// Forward declaration of the gRPC service wrapper
|
||||
|
||||
@@ -27,6 +27,9 @@ service ImGuiTestHarness {
|
||||
rpc GetTestStatus(GetTestStatusRequest) returns (GetTestStatusResponse);
|
||||
rpc ListTests(ListTestsRequest) returns (ListTestsResponse);
|
||||
rpc GetTestResults(GetTestResultsRequest) returns (GetTestResultsResponse);
|
||||
|
||||
// Widget discovery API (IT-06)
|
||||
rpc DiscoverWidgets(DiscoverWidgetsRequest) returns (DiscoverWidgetsResponse);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -222,3 +225,60 @@ message AssertionResult {
|
||||
string actual_value = 4;
|
||||
string error_message = 5;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// DiscoverWidgets - Enumerate discoverable GUI widgets
|
||||
// ============================================================================
|
||||
|
||||
message DiscoverWidgetsRequest {
|
||||
string window_filter = 1; // Optional: Limit to a window name (case-insensitive)
|
||||
WidgetType type_filter = 2; // Optional: Limit to widget type
|
||||
string path_prefix = 3; // Optional: Require widget path to start with prefix
|
||||
bool include_invisible = 4; // Include widgets that are currently not visible
|
||||
bool include_disabled = 5; // Include widgets that are currently disabled
|
||||
}
|
||||
|
||||
enum WidgetType {
|
||||
WIDGET_TYPE_UNSPECIFIED = 0;
|
||||
WIDGET_TYPE_ALL = 1;
|
||||
WIDGET_TYPE_BUTTON = 2;
|
||||
WIDGET_TYPE_INPUT = 3;
|
||||
WIDGET_TYPE_MENU = 4;
|
||||
WIDGET_TYPE_TAB = 5;
|
||||
WIDGET_TYPE_CHECKBOX = 6;
|
||||
WIDGET_TYPE_SLIDER = 7;
|
||||
WIDGET_TYPE_CANVAS = 8;
|
||||
WIDGET_TYPE_SELECTABLE = 9;
|
||||
WIDGET_TYPE_OTHER = 10;
|
||||
}
|
||||
|
||||
message WidgetBounds {
|
||||
float min_x = 1;
|
||||
float min_y = 2;
|
||||
float max_x = 3;
|
||||
float max_y = 4;
|
||||
}
|
||||
|
||||
message DiscoveredWidget {
|
||||
string path = 1; // Full hierarchical path (e.g. Overworld/Toolset/button:Pan)
|
||||
string label = 2; // Human-readable label (e.g. Pan)
|
||||
string type = 3; // Widget type string (button, input, ...)
|
||||
string description = 4; // Description provided by registry (if any)
|
||||
string suggested_action = 5; // Suggested action for automation (e.g. "Click button:Pan")
|
||||
bool visible = 6; // Currently visible in UI
|
||||
bool enabled = 7; // Currently enabled for interaction
|
||||
WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates
|
||||
uint32 widget_id = 9; // ImGui ID (debugging / direct access)
|
||||
}
|
||||
|
||||
message DiscoveredWindow {
|
||||
string name = 1; // Window name (first segment of path)
|
||||
bool visible = 2; // Whether the window is currently visible
|
||||
repeated DiscoveredWidget widgets = 3; // Widgets contained in this window
|
||||
}
|
||||
|
||||
message DiscoverWidgetsResponse {
|
||||
repeated DiscoveredWindow windows = 1;
|
||||
int32 total_widgets = 2; // Total number of widgets returned
|
||||
int64 generated_at_ms = 3; // Snapshot timestamp (Unix ms)
|
||||
}
|
||||
|
||||
246
src/app/core/widget_discovery_service.cc
Normal file
246
src/app/core/widget_discovery_service.cc
Normal file
@@ -0,0 +1,246 @@
|
||||
#include "app/core/widget_discovery_service.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "absl/strings/ascii.h"
|
||||
#include "absl/strings/match.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/time/clock.h"
|
||||
#include "absl/time/time.h"
|
||||
|
||||
namespace yaze {
|
||||
namespace test {
|
||||
namespace {
|
||||
|
||||
struct WindowEntry {
|
||||
int index = -1;
|
||||
bool visible = true;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void WidgetDiscoveryService::CollectWidgets(
|
||||
ImGuiTestContext* ctx, const DiscoverWidgetsRequest& request,
|
||||
DiscoverWidgetsResponse* response) const {
|
||||
if (!response) {
|
||||
return;
|
||||
}
|
||||
|
||||
response->clear_windows();
|
||||
response->set_total_widgets(0);
|
||||
response->set_generated_at_ms(absl::ToUnixMillis(absl::Now()));
|
||||
|
||||
const auto& registry = gui::WidgetIdRegistry::Instance().GetAllWidgets();
|
||||
if (registry.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const std::string window_filter_lower =
|
||||
absl::AsciiStrToLower(std::string(request.window_filter()));
|
||||
const std::string path_prefix_lower =
|
||||
absl::AsciiStrToLower(std::string(request.path_prefix()));
|
||||
const bool include_invisible = request.include_invisible();
|
||||
const bool include_disabled = request.include_disabled();
|
||||
|
||||
std::map<std::string, WindowEntry> window_lookup;
|
||||
int total_widgets = 0;
|
||||
|
||||
for (const auto& [path, info] : registry) {
|
||||
if (!MatchesType(info.type, request.type_filter())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!MatchesPathPrefix(path, path_prefix_lower)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string window_name = ExtractWindowName(path);
|
||||
if (!MatchesWindow(window_name, window_filter_lower)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
|
||||
WindowEntry& entry = it->second;
|
||||
|
||||
if (inserted) {
|
||||
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||
if (ctx) {
|
||||
ImGuiTestItemInfo window_info =
|
||||
ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError);
|
||||
entry.visible = (window_info.ID != 0);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
if (!include_invisible && !entry.visible) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.index == -1) {
|
||||
DiscoveredWindow* window_proto = response->add_windows();
|
||||
entry.index = response->windows_size() - 1;
|
||||
window_proto->set_name(window_name);
|
||||
window_proto->set_visible(entry.visible);
|
||||
}
|
||||
|
||||
const std::string label = ExtractLabel(path);
|
||||
|
||||
bool widget_enabled = true;
|
||||
bool widget_visible = entry.visible;
|
||||
|
||||
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||
bool has_item_info = false;
|
||||
ImGuiTestItemInfo item_info;
|
||||
if (ctx) {
|
||||
item_info = ctx->ItemInfo(label.c_str(), ImGuiTestOpFlags_NoError);
|
||||
if (item_info.ID != 0) {
|
||||
has_item_info = true;
|
||||
widget_visible = item_info.RectClipped.GetWidth() > 0.0f &&
|
||||
item_info.RectClipped.GetHeight() > 0.0f;
|
||||
widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!include_invisible && !widget_visible) {
|
||||
continue;
|
||||
}
|
||||
if (!include_disabled && !widget_enabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* window_proto = response->mutable_windows(entry.index);
|
||||
auto* widget_proto = window_proto->add_widgets();
|
||||
widget_proto->set_path(path);
|
||||
widget_proto->set_label(label);
|
||||
widget_proto->set_type(info.type);
|
||||
widget_proto->set_suggested_action(SuggestedAction(info.type, label));
|
||||
widget_proto->set_visible(widget_visible);
|
||||
widget_proto->set_enabled(widget_enabled);
|
||||
widget_proto->set_widget_id(info.imgui_id);
|
||||
|
||||
if (!info.description.empty()) {
|
||||
widget_proto->set_description(info.description);
|
||||
}
|
||||
|
||||
WidgetBounds* bounds = widget_proto->mutable_bounds();
|
||||
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||
if (ctx && has_item_info) {
|
||||
bounds->set_min_x(item_info.RectFull.Min.x);
|
||||
bounds->set_min_y(item_info.RectFull.Min.y);
|
||||
bounds->set_max_x(item_info.RectFull.Max.x);
|
||||
bounds->set_max_y(item_info.RectFull.Max.y);
|
||||
} else {
|
||||
(void)ctx;
|
||||
}
|
||||
#else
|
||||
(void)ctx;
|
||||
#endif
|
||||
|
||||
++total_widgets;
|
||||
}
|
||||
|
||||
response->set_total_widgets(total_widgets);
|
||||
}
|
||||
|
||||
bool WidgetDiscoveryService::MatchesWindow(absl::string_view window_name,
|
||||
absl::string_view filter_lower) const {
|
||||
if (filter_lower.empty()) {
|
||||
return true;
|
||||
}
|
||||
std::string name_lower = absl::AsciiStrToLower(std::string(window_name));
|
||||
return absl::StrContains(name_lower, filter_lower);
|
||||
}
|
||||
|
||||
bool WidgetDiscoveryService::MatchesPathPrefix(absl::string_view path,
|
||||
absl::string_view prefix_lower) const {
|
||||
if (prefix_lower.empty()) {
|
||||
return true;
|
||||
}
|
||||
std::string path_lower = absl::AsciiStrToLower(std::string(path));
|
||||
return absl::StartsWith(path_lower, prefix_lower);
|
||||
}
|
||||
|
||||
bool WidgetDiscoveryService::MatchesType(absl::string_view type,
|
||||
WidgetType filter) const {
|
||||
if (filter == WIDGET_TYPE_UNSPECIFIED || filter == WIDGET_TYPE_ALL) {
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string type_lower = absl::AsciiStrToLower(std::string(type));
|
||||
switch (filter) {
|
||||
case WIDGET_TYPE_BUTTON:
|
||||
return type_lower == "button" || type_lower == "menuitem";
|
||||
case WIDGET_TYPE_INPUT:
|
||||
return type_lower == "input" || type_lower == "textbox" ||
|
||||
type_lower == "inputtext";
|
||||
case WIDGET_TYPE_MENU:
|
||||
return type_lower == "menu" || type_lower == "menuitem";
|
||||
case WIDGET_TYPE_TAB:
|
||||
return type_lower == "tab";
|
||||
case WIDGET_TYPE_CHECKBOX:
|
||||
return type_lower == "checkbox";
|
||||
case WIDGET_TYPE_SLIDER:
|
||||
return type_lower == "slider" || type_lower == "drag";
|
||||
case WIDGET_TYPE_CANVAS:
|
||||
return type_lower == "canvas";
|
||||
case WIDGET_TYPE_SELECTABLE:
|
||||
return type_lower == "selectable";
|
||||
case WIDGET_TYPE_OTHER:
|
||||
return true;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
std::string WidgetDiscoveryService::ExtractWindowName(
|
||||
absl::string_view path) const {
|
||||
size_t slash = path.find('/');
|
||||
if (slash == absl::string_view::npos) {
|
||||
return std::string(path);
|
||||
}
|
||||
return std::string(path.substr(0, slash));
|
||||
}
|
||||
|
||||
std::string WidgetDiscoveryService::ExtractLabel(absl::string_view path) const {
|
||||
size_t colon = path.rfind(':');
|
||||
if (colon == absl::string_view::npos) {
|
||||
size_t slash = path.rfind('/');
|
||||
if (slash == absl::string_view::npos) {
|
||||
return std::string(path);
|
||||
}
|
||||
return std::string(path.substr(slash + 1));
|
||||
}
|
||||
return std::string(path.substr(colon + 1));
|
||||
}
|
||||
|
||||
std::string WidgetDiscoveryService::SuggestedAction(absl::string_view type,
|
||||
absl::string_view label) const {
|
||||
std::string type_lower = absl::AsciiStrToLower(std::string(type));
|
||||
if (type_lower == "button" || type_lower == "menuitem") {
|
||||
return absl::StrCat("Click button:", label);
|
||||
}
|
||||
if (type_lower == "checkbox") {
|
||||
return absl::StrCat("Toggle checkbox:", label);
|
||||
}
|
||||
if (type_lower == "slider" || type_lower == "drag") {
|
||||
return absl::StrCat("Adjust slider:", label);
|
||||
}
|
||||
if (type_lower == "input" || type_lower == "textbox" ||
|
||||
type_lower == "inputtext") {
|
||||
return absl::StrCat("Type text into:", label);
|
||||
}
|
||||
if (type_lower == "canvas") {
|
||||
return absl::StrCat("Interact with canvas:", label);
|
||||
}
|
||||
if (type_lower == "tab") {
|
||||
return absl::StrCat("Switch to tab:", label);
|
||||
}
|
||||
return absl::StrCat("Interact with:", label);
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace yaze
|
||||
47
src/app/core/widget_discovery_service.h
Normal file
47
src/app/core/widget_discovery_service.h
Normal file
@@ -0,0 +1,47 @@
|
||||
#ifndef YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
|
||||
#define YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "app/core/proto/imgui_test_harness.pb.h"
|
||||
#include "app/gui/widget_id_registry.h"
|
||||
|
||||
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||
#include "imgui_test_engine/imgui_te_context.h"
|
||||
#else
|
||||
struct ImGuiTestContext;
|
||||
#endif
|
||||
|
||||
namespace yaze {
|
||||
namespace test {
|
||||
|
||||
// Service responsible for transforming widget registry data into
|
||||
// DiscoverWidgetsResponse payloads.
|
||||
class WidgetDiscoveryService {
|
||||
public:
|
||||
WidgetDiscoveryService() = default;
|
||||
|
||||
void CollectWidgets(ImGuiTestContext* ctx,
|
||||
const DiscoverWidgetsRequest& request,
|
||||
DiscoverWidgetsResponse* response) const;
|
||||
|
||||
private:
|
||||
bool MatchesWindow(absl::string_view window_name,
|
||||
absl::string_view filter) const;
|
||||
bool MatchesPathPrefix(absl::string_view path,
|
||||
absl::string_view prefix) const;
|
||||
bool MatchesType(absl::string_view type,
|
||||
WidgetType filter) const;
|
||||
|
||||
std::string ExtractWindowName(absl::string_view path) const;
|
||||
std::string ExtractLabel(absl::string_view path) const;
|
||||
std::string SuggestedAction(absl::string_view type,
|
||||
absl::string_view label) const;
|
||||
};
|
||||
|
||||
} // namespace test
|
||||
} // namespace yaze
|
||||
|
||||
#endif // YAZE_APP_CORE_WIDGET_DISCOVERY_SERVICE_H_
|
||||
@@ -25,6 +25,8 @@
|
||||
#include <thread>
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
|
||||
// Declare the rom flag so we can access it
|
||||
ABSL_DECLARE_FLAG(std::string, rom);
|
||||
@@ -105,6 +107,45 @@ std::optional<TestRunStatus> ParseStatusFilter(absl::string_view value) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<WidgetTypeFilter> ParseWidgetTypeFilter(
|
||||
absl::string_view value) {
|
||||
std::string lower = std::string(absl::AsciiStrToLower(value));
|
||||
if (lower.empty() || lower == "unspecified" || lower == "any") {
|
||||
return WidgetTypeFilter::kUnspecified;
|
||||
}
|
||||
if (lower == "all") {
|
||||
return WidgetTypeFilter::kAll;
|
||||
}
|
||||
if (lower == "button" || lower == "buttons") {
|
||||
return WidgetTypeFilter::kButton;
|
||||
}
|
||||
if (lower == "input" || lower == "textbox" || lower == "field") {
|
||||
return WidgetTypeFilter::kInput;
|
||||
}
|
||||
if (lower == "menu" || lower == "menuitem" || lower == "menu-item") {
|
||||
return WidgetTypeFilter::kMenu;
|
||||
}
|
||||
if (lower == "tab" || lower == "tabs") {
|
||||
return WidgetTypeFilter::kTab;
|
||||
}
|
||||
if (lower == "checkbox" || lower == "toggle") {
|
||||
return WidgetTypeFilter::kCheckbox;
|
||||
}
|
||||
if (lower == "slider" || lower == "drag" || lower == "sliderfloat") {
|
||||
return WidgetTypeFilter::kSlider;
|
||||
}
|
||||
if (lower == "canvas" || lower == "viewport") {
|
||||
return WidgetTypeFilter::kCanvas;
|
||||
}
|
||||
if (lower == "selectable" || lower == "list-item") {
|
||||
return WidgetTypeFilter::kSelectable;
|
||||
}
|
||||
if (lower == "other") {
|
||||
return WidgetTypeFilter::kOther;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::string HarnessAddress(const std::string& host, int port) {
|
||||
return absl::StrFormat("%s:%d", host, port);
|
||||
}
|
||||
@@ -1051,6 +1092,327 @@ absl::Status HandleTestCommand(const std::vector<std::string>& arg_vec) {
|
||||
return HandleTestRunCommand(arg_vec);
|
||||
}
|
||||
|
||||
absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
|
||||
std::string host = "localhost";
|
||||
int port = 50052;
|
||||
std::string window_filter;
|
||||
std::string path_prefix;
|
||||
std::optional<WidgetTypeFilter> type_filter;
|
||||
std::optional<std::string> type_filter_label;
|
||||
bool include_invisible = false;
|
||||
bool include_disabled = false;
|
||||
std::string format = "table";
|
||||
int limit = -1;
|
||||
|
||||
auto require_value = [&](const std::vector<std::string>& args, size_t& index,
|
||||
absl::string_view flag) -> absl::StatusOr<std::string> {
|
||||
if (index + 1 >= args.size()) {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Flag %s requires a value", flag));
|
||||
}
|
||||
return args[++index];
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < arg_vec.size(); ++i) {
|
||||
const std::string& token = arg_vec[i];
|
||||
if (token == "--host") {
|
||||
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--host"));
|
||||
host = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--host=")) {
|
||||
host = token.substr(7);
|
||||
} else if (token == "--port") {
|
||||
ASSIGN_OR_RETURN(auto value, require_value(arg_vec, i, "--port"));
|
||||
port = std::stoi(value);
|
||||
} else if (absl::StartsWith(token, "--port=")) {
|
||||
port = std::stoi(token.substr(7));
|
||||
} else if (token == "--window" || token == "--window-filter") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, token.c_str()));
|
||||
window_filter = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--window=")) {
|
||||
window_filter = token.substr(9);
|
||||
} else if (token == "--path-prefix") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--path-prefix"));
|
||||
path_prefix = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--path-prefix=")) {
|
||||
path_prefix = token.substr(14);
|
||||
} else if (token == "--type") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--type"));
|
||||
auto parsed = ParseWidgetTypeFilter(value);
|
||||
if (!parsed.has_value()) {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown widget type filter: %s", value));
|
||||
}
|
||||
type_filter = parsed;
|
||||
type_filter_label = absl::AsciiStrToLower(value);
|
||||
} else if (absl::StartsWith(token, "--type=")) {
|
||||
std::string value = token.substr(7);
|
||||
auto parsed = ParseWidgetTypeFilter(value);
|
||||
if (!parsed.has_value()) {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown widget type filter: %s", value));
|
||||
}
|
||||
type_filter = parsed;
|
||||
type_filter_label = absl::AsciiStrToLower(value);
|
||||
} else if (token == "--include-invisible") {
|
||||
include_invisible = true;
|
||||
} else if (token == "--include-disabled") {
|
||||
include_disabled = true;
|
||||
} else if (token == "--format") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--format"));
|
||||
format = std::move(value);
|
||||
} else if (absl::StartsWith(token, "--format=")) {
|
||||
format = token.substr(9);
|
||||
} else if (token == "--limit") {
|
||||
ASSIGN_OR_RETURN(auto value,
|
||||
require_value(arg_vec, i, "--limit"));
|
||||
limit = std::stoi(value);
|
||||
} else if (absl::StartsWith(token, "--limit=")) {
|
||||
limit = std::stoi(token.substr(8));
|
||||
} else if (token == "--help" || token == "-h") {
|
||||
std::cout << "Usage: agent gui discover [options]\n"
|
||||
<< " --host <host>\n"
|
||||
<< " --port <port>\n"
|
||||
<< " --window <name>\n"
|
||||
<< " --type <widget-type>\n"
|
||||
<< " --path-prefix <path>\n"
|
||||
<< " --include-invisible\n"
|
||||
<< " --include-disabled\n"
|
||||
<< " --format <table|json>\n"
|
||||
<< " --limit <n>\n";
|
||||
return absl::OkStatus();
|
||||
} else {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown flag for agent gui discover: %s",
|
||||
token));
|
||||
}
|
||||
}
|
||||
|
||||
format = absl::AsciiStrToLower(format);
|
||||
if (format != "table" && format != "json") {
|
||||
return absl::InvalidArgumentError(
|
||||
"--format must be either 'table' or 'json'");
|
||||
}
|
||||
|
||||
if (limit == 0) {
|
||||
return absl::InvalidArgumentError("--limit must be positive");
|
||||
}
|
||||
|
||||
#ifndef YAZE_WITH_GRPC
|
||||
(void)host;
|
||||
(void)port;
|
||||
(void)window_filter;
|
||||
(void)path_prefix;
|
||||
(void)type_filter;
|
||||
(void)include_invisible;
|
||||
(void)include_disabled;
|
||||
(void)format;
|
||||
(void)limit;
|
||||
return absl::UnimplementedError(
|
||||
"GUI automation requires YAZE_WITH_GRPC=ON at build time.\n"
|
||||
"Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON");
|
||||
#else
|
||||
GuiAutomationClient client(HarnessAddress(host, port));
|
||||
RETURN_IF_ERROR(client.Connect());
|
||||
|
||||
DiscoverWidgetsQuery query;
|
||||
query.window_filter = window_filter;
|
||||
query.path_prefix = path_prefix;
|
||||
if (type_filter.has_value()) {
|
||||
query.type_filter = type_filter.value();
|
||||
}
|
||||
query.include_invisible = include_invisible;
|
||||
query.include_disabled = include_disabled;
|
||||
|
||||
ASSIGN_OR_RETURN(auto response, client.DiscoverWidgets(query));
|
||||
|
||||
int max_items = limit > 0 ? limit : std::numeric_limits<int>::max();
|
||||
int remaining = max_items;
|
||||
std::vector<DiscoveredWindowInfo> trimmed_windows;
|
||||
trimmed_windows.reserve(response.windows.size());
|
||||
int rendered_widgets = 0;
|
||||
|
||||
for (const auto& window : response.windows) {
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
DiscoveredWindowInfo trimmed;
|
||||
trimmed.name = window.name;
|
||||
trimmed.visible = window.visible;
|
||||
|
||||
for (const auto& widget : window.widgets) {
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
trimmed.widgets.push_back(widget);
|
||||
--remaining;
|
||||
++rendered_widgets;
|
||||
}
|
||||
|
||||
if (!trimmed.widgets.empty()) {
|
||||
trimmed_windows.push_back(std::move(trimmed));
|
||||
}
|
||||
}
|
||||
|
||||
bool truncated = rendered_widgets < response.total_widgets;
|
||||
|
||||
if (format == "json") {
|
||||
std::cout << "{\n";
|
||||
std::cout << " \"server\": \""
|
||||
<< JsonEscape(HarnessAddress(host, port)) << "\",\n";
|
||||
std::cout << " \"totalWidgets\": " << response.total_widgets << ",\n";
|
||||
std::cout << " \"returnedWidgets\": " << rendered_widgets << ",\n";
|
||||
std::cout << " \"truncated\": " << (truncated ? "true" : "false")
|
||||
<< ",\n";
|
||||
std::cout << " \"generatedAt\": "
|
||||
<< (response.generated_at.has_value()
|
||||
? absl::StrCat("\"",
|
||||
JsonEscape(absl::FormatTime(
|
||||
"%Y-%m-%dT%H:%M:%SZ",
|
||||
*response.generated_at,
|
||||
absl::UTCTimeZone())),
|
||||
"\"")
|
||||
: std::string("null"))
|
||||
<< ",\n";
|
||||
std::cout << " \"windows\": [\n";
|
||||
|
||||
for (size_t w = 0; w < trimmed_windows.size(); ++w) {
|
||||
const auto& window = trimmed_windows[w];
|
||||
std::cout << " {\n";
|
||||
std::cout << " \"name\": \"" << JsonEscape(window.name)
|
||||
<< "\",\n";
|
||||
std::cout << " \"visible\": "
|
||||
<< (window.visible ? "true" : "false") << ",\n";
|
||||
std::cout << " \"widgets\": [\n";
|
||||
for (size_t i = 0; i < window.widgets.size(); ++i) {
|
||||
const auto& widget = window.widgets[i];
|
||||
std::cout << " {\n";
|
||||
std::cout << " \"path\": \""
|
||||
<< JsonEscape(widget.path) << "\",\n";
|
||||
std::cout << " \"label\": \""
|
||||
<< JsonEscape(widget.label) << "\",\n";
|
||||
std::cout << " \"type\": \""
|
||||
<< JsonEscape(widget.type) << "\",\n";
|
||||
std::cout << " \"description\": \""
|
||||
<< JsonEscape(widget.description) << "\",\n";
|
||||
std::cout << " \"suggestedAction\": \""
|
||||
<< JsonEscape(widget.suggested_action) << "\",\n";
|
||||
std::cout << " \"visible\": "
|
||||
<< (widget.visible ? "true" : "false") << ",\n";
|
||||
std::cout << " \"enabled\": "
|
||||
<< (widget.enabled ? "true" : "false") << ",\n";
|
||||
std::cout << " \"bounds\": { \"min\": ["
|
||||
<< widget.bounds.min_x << ", "
|
||||
<< widget.bounds.min_y << "], \"max\": ["
|
||||
<< widget.bounds.max_x << ", "
|
||||
<< widget.bounds.max_y << "] },\n";
|
||||
std::cout << " \"widgetId\": " << widget.widget_id
|
||||
<< "\n";
|
||||
std::cout << " }";
|
||||
if (i + 1 < window.widgets.size()) {
|
||||
std::cout << ",";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
std::cout << " ]\n";
|
||||
std::cout << " }";
|
||||
if (w + 1 < trimmed_windows.size()) {
|
||||
std::cout << ",";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
|
||||
std::cout << " ]\n";
|
||||
std::cout << "}\n";
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
std::cout << "\n=== Widget Discovery ===\n";
|
||||
std::cout << "Server: " << HarnessAddress(host, port) << "\n";
|
||||
if (!window_filter.empty()) {
|
||||
std::cout << "Window filter: " << window_filter << "\n";
|
||||
}
|
||||
if (!path_prefix.empty()) {
|
||||
std::cout << "Path prefix: " << path_prefix << "\n";
|
||||
}
|
||||
if (type_filter_label.has_value()) {
|
||||
std::cout << "Type filter: " << *type_filter_label << "\n";
|
||||
}
|
||||
std::cout << "Include invisible: " << (include_invisible ? "yes" : "no")
|
||||
<< "\n";
|
||||
std::cout << "Include disabled: " << (include_disabled ? "yes" : "no")
|
||||
<< "\n\n";
|
||||
|
||||
if (trimmed_windows.empty()) {
|
||||
std::cout << "No widgets matched the provided filters." << std::endl;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
for (const auto& window : trimmed_windows) {
|
||||
std::cout << "Window: " << window.name
|
||||
<< (window.visible ? " (visible)" : " (hidden)")
|
||||
<< "\n";
|
||||
for (const auto& widget : window.widgets) {
|
||||
std::cout << " • [" << widget.type << "] " << widget.label
|
||||
<< "\n";
|
||||
std::cout << " Path: " << widget.path << "\n";
|
||||
if (!widget.description.empty()) {
|
||||
std::cout << " Description: " << widget.description
|
||||
<< "\n";
|
||||
}
|
||||
std::cout << " Suggested: " << widget.suggested_action << "\n";
|
||||
std::cout << " State: "
|
||||
<< (widget.visible ? "visible" : "hidden") << ", "
|
||||
<< (widget.enabled ? "enabled" : "disabled") << "\n";
|
||||
std::cout << absl::StrFormat(
|
||||
" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n",
|
||||
widget.bounds.min_x, widget.bounds.min_y,
|
||||
widget.bounds.max_x, widget.bounds.max_y);
|
||||
std::cout << " Widget ID: 0x" << std::hex << widget.widget_id
|
||||
<< std::dec << "\n";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
|
||||
std::cout << "Widgets shown: " << rendered_widgets << " of "
|
||||
<< response.total_widgets;
|
||||
if (truncated) {
|
||||
std::cout << " (truncated)";
|
||||
}
|
||||
std::cout << "\n";
|
||||
|
||||
if (response.generated_at.has_value()) {
|
||||
std::cout << "Snapshot: "
|
||||
<< absl::FormatTime("%Y-%m-%d %H:%M:%S",
|
||||
*response.generated_at,
|
||||
absl::LocalTimeZone())
|
||||
<< "\n";
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::Status HandleGuiCommand(const std::vector<std::string>& arg_vec) {
|
||||
if (arg_vec.empty()) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Usage: agent gui <discover> [options]");
|
||||
}
|
||||
|
||||
const std::string& subcommand = arg_vec[0];
|
||||
std::vector<std::string> tail(arg_vec.begin() + 1, arg_vec.end());
|
||||
|
||||
if (subcommand == "discover") {
|
||||
return HandleGuiDiscoverCommand(tail);
|
||||
}
|
||||
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat("Unknown agent gui subcommand: %s", subcommand));
|
||||
}
|
||||
|
||||
absl::Status HandleLearnCommand() {
|
||||
std::cout << "Agent learn not yet implemented." << std::endl;
|
||||
return absl::OkStatus();
|
||||
@@ -1184,7 +1546,7 @@ absl::Status HandleDescribeCommand(const std::vector<std::string>& arg_vec) {
|
||||
absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
|
||||
if (arg_vec.empty()) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Usage: agent <run|plan|diff|test|learn|list|commit|revert|describe> [options]");
|
||||
"Usage: agent <run|plan|diff|test|gui|learn|list|commit|revert|describe> [options]");
|
||||
}
|
||||
|
||||
std::string subcommand = arg_vec[0];
|
||||
@@ -1198,6 +1560,8 @@ absl::Status Agent::Run(const std::vector<std::string>& arg_vec) {
|
||||
return HandleDiffCommand(rom_, subcommand_args);
|
||||
} else if (subcommand == "test") {
|
||||
return HandleTestCommand(subcommand_args);
|
||||
} else if (subcommand == "gui") {
|
||||
return HandleGuiCommand(subcommand_args);
|
||||
} else if (subcommand == "learn") {
|
||||
return HandleLearnCommand();
|
||||
} else if (subcommand == "list") {
|
||||
|
||||
@@ -60,11 +60,13 @@ void ModernCLI::SetupCommands() {
|
||||
commands_["agent"] = {
|
||||
.name = "agent",
|
||||
.description = "Interact with the AI agent",
|
||||
.usage = "z3ed agent <run|plan|diff|test|list|learn|commit|revert|describe> [options]\n"
|
||||
.usage = "z3ed agent <run|plan|diff|test|gui|list|learn|commit|revert|describe> [options]\n"
|
||||
" test run: --prompt \"<description>\" [--host <host>] [--port <port>] [--timeout <sec>]\n"
|
||||
" test status: status --test-id <id> [--follow] [--host <host>] [--port <port>]\n"
|
||||
" test list: list [--category <name>] [--status <state>] [--limit <n>] [--host <host>] [--port <port>]\n"
|
||||
" test results: results --test-id <id> [--include-logs] [--format yaml|json] [--host <host>] [--port <port>]\n"
|
||||
" gui discover: discover [--window <name>] [--type <widget>] [--path-prefix <path>]\n"
|
||||
" [--include-invisible] [--include-disabled] [--format table|json] [--limit <n>]\n"
|
||||
" describe options: [--resource <name>] [--format json|yaml] [--output <path>]\n"
|
||||
" [--version <value>] [--last-updated <date>]",
|
||||
.handler = [this](const std::vector<std::string>& args) -> absl::Status {
|
||||
|
||||
@@ -21,6 +21,35 @@ std::optional<absl::Time> OptionalTimeFromMillis(int64_t millis) {
|
||||
return absl::FromUnixMillis(millis);
|
||||
}
|
||||
|
||||
yaze::test::WidgetType ConvertWidgetTypeFilterToProto(WidgetTypeFilter filter) {
|
||||
using ProtoType = yaze::test::WidgetType;
|
||||
switch (filter) {
|
||||
case WidgetTypeFilter::kAll:
|
||||
return ProtoType::WIDGET_TYPE_ALL;
|
||||
case WidgetTypeFilter::kButton:
|
||||
return ProtoType::WIDGET_TYPE_BUTTON;
|
||||
case WidgetTypeFilter::kInput:
|
||||
return ProtoType::WIDGET_TYPE_INPUT;
|
||||
case WidgetTypeFilter::kMenu:
|
||||
return ProtoType::WIDGET_TYPE_MENU;
|
||||
case WidgetTypeFilter::kTab:
|
||||
return ProtoType::WIDGET_TYPE_TAB;
|
||||
case WidgetTypeFilter::kCheckbox:
|
||||
return ProtoType::WIDGET_TYPE_CHECKBOX;
|
||||
case WidgetTypeFilter::kSlider:
|
||||
return ProtoType::WIDGET_TYPE_SLIDER;
|
||||
case WidgetTypeFilter::kCanvas:
|
||||
return ProtoType::WIDGET_TYPE_CANVAS;
|
||||
case WidgetTypeFilter::kSelectable:
|
||||
return ProtoType::WIDGET_TYPE_SELECTABLE;
|
||||
case WidgetTypeFilter::kOther:
|
||||
return ProtoType::WIDGET_TYPE_OTHER;
|
||||
case WidgetTypeFilter::kUnspecified:
|
||||
default:
|
||||
return ProtoType::WIDGET_TYPE_UNSPECIFIED;
|
||||
}
|
||||
}
|
||||
|
||||
TestRunStatus ConvertStatusProto(
|
||||
yaze::test::GetTestStatusResponse::Status status) {
|
||||
using ProtoStatus = yaze::test::GetTestStatusResponse::Status;
|
||||
@@ -440,5 +469,73 @@ absl::StatusOr<TestResultDetails> GuiAutomationClient::GetTestResults(
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<DiscoverWidgetsResult> GuiAutomationClient::DiscoverWidgets(
|
||||
const DiscoverWidgetsQuery& query) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::DiscoverWidgetsRequest request;
|
||||
if (!query.window_filter.empty()) {
|
||||
request.set_window_filter(query.window_filter);
|
||||
}
|
||||
request.set_type_filter(ConvertWidgetTypeFilterToProto(query.type_filter));
|
||||
if (!query.path_prefix.empty()) {
|
||||
request.set_path_prefix(query.path_prefix);
|
||||
}
|
||||
request.set_include_invisible(query.include_invisible);
|
||||
request.set_include_disabled(query.include_disabled);
|
||||
|
||||
yaze::test::DiscoverWidgetsResponse response;
|
||||
grpc::ClientContext context;
|
||||
grpc::Status status = stub_->DiscoverWidgets(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("DiscoverWidgets RPC failed: %s",
|
||||
status.error_message()));
|
||||
}
|
||||
|
||||
DiscoverWidgetsResult result;
|
||||
result.total_widgets = response.total_widgets();
|
||||
if (response.generated_at_ms() > 0) {
|
||||
result.generated_at = OptionalTimeFromMillis(response.generated_at_ms());
|
||||
}
|
||||
|
||||
result.windows.reserve(response.windows_size());
|
||||
for (const auto& window_proto : response.windows()) {
|
||||
DiscoveredWindowInfo window_info;
|
||||
window_info.name = window_proto.name();
|
||||
window_info.visible = window_proto.visible();
|
||||
window_info.widgets.reserve(window_proto.widgets_size());
|
||||
|
||||
for (const auto& widget_proto : window_proto.widgets()) {
|
||||
WidgetDescriptor widget;
|
||||
widget.path = widget_proto.path();
|
||||
widget.label = widget_proto.label();
|
||||
widget.type = widget_proto.type();
|
||||
widget.description = widget_proto.description();
|
||||
widget.suggested_action = widget_proto.suggested_action();
|
||||
widget.visible = widget_proto.visible();
|
||||
widget.enabled = widget_proto.enabled();
|
||||
widget.bounds.min_x = widget_proto.bounds().min_x();
|
||||
widget.bounds.min_y = widget_proto.bounds().min_y();
|
||||
widget.bounds.max_x = widget_proto.bounds().max_x();
|
||||
widget.bounds.max_y = widget_proto.bounds().max_y();
|
||||
widget.widget_id = widget_proto.widget_id();
|
||||
window_info.widgets.push_back(std::move(widget));
|
||||
}
|
||||
|
||||
result.windows.push_back(std::move(window_info));
|
||||
}
|
||||
|
||||
return result;
|
||||
#else
|
||||
(void)query;
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "absl/time/time.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
@@ -120,6 +121,59 @@ struct TestResultDetails {
|
||||
std::map<std::string, int> metrics;
|
||||
};
|
||||
|
||||
enum class WidgetTypeFilter {
|
||||
kUnspecified,
|
||||
kAll,
|
||||
kButton,
|
||||
kInput,
|
||||
kMenu,
|
||||
kTab,
|
||||
kCheckbox,
|
||||
kSlider,
|
||||
kCanvas,
|
||||
kSelectable,
|
||||
kOther,
|
||||
};
|
||||
|
||||
struct WidgetBoundingBox {
|
||||
float min_x = 0.0f;
|
||||
float min_y = 0.0f;
|
||||
float max_x = 0.0f;
|
||||
float max_y = 0.0f;
|
||||
};
|
||||
|
||||
struct WidgetDescriptor {
|
||||
std::string path;
|
||||
std::string label;
|
||||
std::string type;
|
||||
std::string description;
|
||||
std::string suggested_action;
|
||||
bool visible = true;
|
||||
bool enabled = true;
|
||||
WidgetBoundingBox bounds;
|
||||
uint32_t widget_id = 0;
|
||||
};
|
||||
|
||||
struct DiscoveredWindowInfo {
|
||||
std::string name;
|
||||
bool visible = true;
|
||||
std::vector<WidgetDescriptor> widgets;
|
||||
};
|
||||
|
||||
struct DiscoverWidgetsQuery {
|
||||
std::string window_filter;
|
||||
WidgetTypeFilter type_filter = WidgetTypeFilter::kUnspecified;
|
||||
std::string path_prefix;
|
||||
bool include_invisible = false;
|
||||
bool include_disabled = false;
|
||||
};
|
||||
|
||||
struct DiscoverWidgetsResult {
|
||||
std::vector<DiscoveredWindowInfo> windows;
|
||||
int total_widgets = 0;
|
||||
std::optional<absl::Time> generated_at;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Client for automating YAZE GUI through gRPC
|
||||
*
|
||||
@@ -225,6 +279,9 @@ class GuiAutomationClient {
|
||||
absl::StatusOr<TestResultDetails> GetTestResults(const std::string& test_id,
|
||||
bool include_logs = false);
|
||||
|
||||
absl::StatusOr<DiscoverWidgetsResult> DiscoverWidgets(
|
||||
const DiscoverWidgetsQuery& query);
|
||||
|
||||
/**
|
||||
* @brief Check if client is connected
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user