Files
yaze/src/app/core/widget_discovery_service.cc
scawful 22f0e5006b feat: Implement widget discovery feature in GUI automation
- Added `DiscoverWidgets` RPC to the ImGuiTestHarness service for enumerating GUI widgets.
- Introduced `WidgetDiscoveryService` to handle widget collection and filtering based on various criteria.
- Updated `agent gui discover` command to support new options for filtering and output formats.
- Enhanced `GuiAutomationClient` to facilitate widget discovery requests and responses.
- Added necessary protobuf messages for widget discovery in `imgui_test_harness.proto`.
- Updated CLI command handling to include new GUI discovery functionality.
- Improved documentation for the `agent gui discover` command with examples and output formats.
2025-10-02 16:56:15 -04:00

247 lines
7.4 KiB
C++

#include "app/core/widget_discovery_service.h"
#include <algorithm>
#include <map>
#include <string>
#include <utility>
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
namespace yaze {
namespace test {
namespace {
struct WindowEntry {
int index = -1;
bool visible = true;
};
} // namespace
void WidgetDiscoveryService::CollectWidgets(
ImGuiTestContext* ctx, const DiscoverWidgetsRequest& request,
DiscoverWidgetsResponse* response) const {
if (!response) {
return;
}
response->clear_windows();
response->set_total_widgets(0);
response->set_generated_at_ms(absl::ToUnixMillis(absl::Now()));
const auto& registry = gui::WidgetIdRegistry::Instance().GetAllWidgets();
if (registry.empty()) {
return;
}
const std::string window_filter_lower =
absl::AsciiStrToLower(std::string(request.window_filter()));
const std::string path_prefix_lower =
absl::AsciiStrToLower(std::string(request.path_prefix()));
const bool include_invisible = request.include_invisible();
const bool include_disabled = request.include_disabled();
std::map<std::string, WindowEntry> window_lookup;
int total_widgets = 0;
for (const auto& [path, info] : registry) {
if (!MatchesType(info.type, request.type_filter())) {
continue;
}
if (!MatchesPathPrefix(path, path_prefix_lower)) {
continue;
}
const std::string window_name = ExtractWindowName(path);
if (!MatchesWindow(window_name, window_filter_lower)) {
continue;
}
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
WindowEntry& entry = it->second;
if (inserted) {
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
if (ctx) {
ImGuiTestItemInfo window_info =
ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError);
entry.visible = (window_info.ID != 0);
}
#endif
}
if (!include_invisible && !entry.visible) {
continue;
}
if (entry.index == -1) {
DiscoveredWindow* window_proto = response->add_windows();
entry.index = response->windows_size() - 1;
window_proto->set_name(window_name);
window_proto->set_visible(entry.visible);
}
const std::string label = ExtractLabel(path);
bool widget_enabled = true;
bool widget_visible = entry.visible;
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
bool has_item_info = false;
ImGuiTestItemInfo item_info;
if (ctx) {
item_info = ctx->ItemInfo(label.c_str(), ImGuiTestOpFlags_NoError);
if (item_info.ID != 0) {
has_item_info = true;
widget_visible = item_info.RectClipped.GetWidth() > 0.0f &&
item_info.RectClipped.GetHeight() > 0.0f;
widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0;
}
}
#endif
if (!include_invisible && !widget_visible) {
continue;
}
if (!include_disabled && !widget_enabled) {
continue;
}
auto* window_proto = response->mutable_windows(entry.index);
auto* widget_proto = window_proto->add_widgets();
widget_proto->set_path(path);
widget_proto->set_label(label);
widget_proto->set_type(info.type);
widget_proto->set_suggested_action(SuggestedAction(info.type, label));
widget_proto->set_visible(widget_visible);
widget_proto->set_enabled(widget_enabled);
widget_proto->set_widget_id(info.imgui_id);
if (!info.description.empty()) {
widget_proto->set_description(info.description);
}
WidgetBounds* bounds = widget_proto->mutable_bounds();
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
if (ctx && has_item_info) {
bounds->set_min_x(item_info.RectFull.Min.x);
bounds->set_min_y(item_info.RectFull.Min.y);
bounds->set_max_x(item_info.RectFull.Max.x);
bounds->set_max_y(item_info.RectFull.Max.y);
} else {
(void)ctx;
}
#else
(void)ctx;
#endif
++total_widgets;
}
response->set_total_widgets(total_widgets);
}
bool WidgetDiscoveryService::MatchesWindow(absl::string_view window_name,
absl::string_view filter_lower) const {
if (filter_lower.empty()) {
return true;
}
std::string name_lower = absl::AsciiStrToLower(std::string(window_name));
return absl::StrContains(name_lower, filter_lower);
}
bool WidgetDiscoveryService::MatchesPathPrefix(absl::string_view path,
absl::string_view prefix_lower) const {
if (prefix_lower.empty()) {
return true;
}
std::string path_lower = absl::AsciiStrToLower(std::string(path));
return absl::StartsWith(path_lower, prefix_lower);
}
bool WidgetDiscoveryService::MatchesType(absl::string_view type,
WidgetType filter) const {
if (filter == WIDGET_TYPE_UNSPECIFIED || filter == WIDGET_TYPE_ALL) {
return true;
}
std::string type_lower = absl::AsciiStrToLower(std::string(type));
switch (filter) {
case WIDGET_TYPE_BUTTON:
return type_lower == "button" || type_lower == "menuitem";
case WIDGET_TYPE_INPUT:
return type_lower == "input" || type_lower == "textbox" ||
type_lower == "inputtext";
case WIDGET_TYPE_MENU:
return type_lower == "menu" || type_lower == "menuitem";
case WIDGET_TYPE_TAB:
return type_lower == "tab";
case WIDGET_TYPE_CHECKBOX:
return type_lower == "checkbox";
case WIDGET_TYPE_SLIDER:
return type_lower == "slider" || type_lower == "drag";
case WIDGET_TYPE_CANVAS:
return type_lower == "canvas";
case WIDGET_TYPE_SELECTABLE:
return type_lower == "selectable";
case WIDGET_TYPE_OTHER:
return true;
default:
return true;
}
}
std::string WidgetDiscoveryService::ExtractWindowName(
absl::string_view path) const {
size_t slash = path.find('/');
if (slash == absl::string_view::npos) {
return std::string(path);
}
return std::string(path.substr(0, slash));
}
std::string WidgetDiscoveryService::ExtractLabel(absl::string_view path) const {
size_t colon = path.rfind(':');
if (colon == absl::string_view::npos) {
size_t slash = path.rfind('/');
if (slash == absl::string_view::npos) {
return std::string(path);
}
return std::string(path.substr(slash + 1));
}
return std::string(path.substr(colon + 1));
}
std::string WidgetDiscoveryService::SuggestedAction(absl::string_view type,
absl::string_view label) const {
std::string type_lower = absl::AsciiStrToLower(std::string(type));
if (type_lower == "button" || type_lower == "menuitem") {
return absl::StrCat("Click button:", label);
}
if (type_lower == "checkbox") {
return absl::StrCat("Toggle checkbox:", label);
}
if (type_lower == "slider" || type_lower == "drag") {
return absl::StrCat("Adjust slider:", label);
}
if (type_lower == "input" || type_lower == "textbox" ||
type_lower == "inputtext") {
return absl::StrCat("Type text into:", label);
}
if (type_lower == "canvas") {
return absl::StrCat("Interact with canvas:", label);
}
if (type_lower == "tab") {
return absl::StrCat("Switch to tab:", label);
}
return absl::StrCat("Interact with:", label);
}
} // namespace test
} // namespace yaze