feat: Implement widget discovery feature in GUI automation

- Added `DiscoverWidgets` RPC to the ImGuiTestHarness service for enumerating GUI widgets.
- Introduced `WidgetDiscoveryService` to handle widget collection and filtering based on various criteria.
- Updated `agent gui discover` command to support new options for filtering and output formats.
- Enhanced `GuiAutomationClient` to facilitate widget discovery requests and responses.
- Added necessary protobuf messages for widget discovery in `imgui_test_harness.proto`.
- Updated CLI command handling to include new GUI discovery functionality.
- Improved documentation for the `agent gui discover` command with examples and output formats.
This commit is contained in:
scawful
2025-10-02 16:56:15 -04:00
parent 3944861b38
commit 22f0e5006b
11 changed files with 947 additions and 31 deletions

View File

@@ -9,6 +9,7 @@
#include "absl/time/time.h"
#include <chrono>
#include <cstdint>
#include <map>
#include <memory>
#include <optional>
@@ -120,6 +121,59 @@ struct TestResultDetails {
std::map<std::string, int> metrics;
};
enum class WidgetTypeFilter {
kUnspecified,
kAll,
kButton,
kInput,
kMenu,
kTab,
kCheckbox,
kSlider,
kCanvas,
kSelectable,
kOther,
};
struct WidgetBoundingBox {
float min_x = 0.0f;
float min_y = 0.0f;
float max_x = 0.0f;
float max_y = 0.0f;
};
struct WidgetDescriptor {
std::string path;
std::string label;
std::string type;
std::string description;
std::string suggested_action;
bool visible = true;
bool enabled = true;
WidgetBoundingBox bounds;
uint32_t widget_id = 0;
};
struct DiscoveredWindowInfo {
std::string name;
bool visible = true;
std::vector<WidgetDescriptor> widgets;
};
struct DiscoverWidgetsQuery {
std::string window_filter;
WidgetTypeFilter type_filter = WidgetTypeFilter::kUnspecified;
std::string path_prefix;
bool include_invisible = false;
bool include_disabled = false;
};
struct DiscoverWidgetsResult {
std::vector<DiscoveredWindowInfo> windows;
int total_widgets = 0;
std::optional<absl::Time> generated_at;
};
/**
* @brief Client for automating YAZE GUI through gRPC
*
@@ -225,6 +279,9 @@ class GuiAutomationClient {
absl::StatusOr<TestResultDetails> GetTestResults(const std::string& test_id,
bool include_logs = false);
absl::StatusOr<DiscoverWidgetsResult> DiscoverWidgets(
const DiscoverWidgetsQuery& query);
/**
* @brief Check if client is connected
*/