From c202aa9fa4d9fc7a0d57962688e6afe3898cf7a1 Mon Sep 17 00:00:00 2001 From: scawful Date: Thu, 2 Oct 2025 22:50:47 -0400 Subject: [PATCH] feat: Enhance widget discovery with telemetry data and improve output formatting --- docs/z3ed/E6-z3ed-implementation-plan.md | 55 ++-- docs/z3ed/E6-z3ed-reference.md | 9 + docs/z3ed/QUICK_REFERENCE.md | 8 +- src/app/core/controller.cc | 6 +- src/app/core/proto/imgui_test_harness.proto | 3 + .../core/service/widget_discovery_service.cc | 79 +++--- src/app/gui/widget_id_registry.cc | 258 +++++++++++++++++- src/app/gui/widget_id_registry.h | 46 +++- src/cli/handlers/agent/gui_commands.cc | 51 +++- src/cli/service/gui_automation_client.cc | 16 +- src/cli/service/gui_automation_client.h | 4 + 11 files changed, 457 insertions(+), 78 deletions(-) diff --git a/docs/z3ed/E6-z3ed-implementation-plan.md b/docs/z3ed/E6-z3ed-implementation-plan.md index 90df0878..913663d9 100644 --- a/docs/z3ed/E6-z3ed-implementation-plan.md +++ b/docs/z3ed/E6-z3ed-implementation-plan.md @@ -202,31 +202,48 @@ z3ed gui discover --window "Overworld" z3ed gui discover --type button ``` -**API Schema**: +**API Schema (current)**: ```proto message DiscoverWidgetsRequest { - string window_filter = 1; // Optional: filter by window name - enum WidgetType { ALL = 0; BUTTON = 1; INPUT = 2; MENU = 3; TAB = 4; CHECKBOX = 5; } + string window_filter = 1; WidgetType type_filter = 2; + string path_prefix = 3; + bool include_invisible = 4; + bool include_disabled = 5; +} + +message WidgetBounds { + float min_x = 1; + float min_y = 2; + float max_x = 3; + float max_y = 4; +} + +message DiscoveredWidget { + string path = 1; + string label = 2; + string type = 3; + string description = 4; + string suggested_action = 5; + bool visible = 6; + bool enabled = 7; + WidgetBounds bounds = 8; + uint32 widget_id = 9; + int64 last_seen_frame = 10; + int64 last_seen_at_ms = 11; + bool stale = 12; +} + +message DiscoveredWindow { + string name = 1; + bool visible = 2; + repeated DiscoveredWidget widgets = 3; } message DiscoverWidgetsResponse { - repeated WindowInfo windows = 1; -} - -message WindowInfo { - string name = 1; - bool is_visible = 2; - repeated WidgetInfo widgets = 3; -} - -message WidgetInfo { - string id = 1; - string label = 2; - string type = 3; // "button", "input", "menu", etc. - bool is_enabled = 4; - string position = 5; // "x,y,width,height" - string suggested_action = 6; // "Click button:Open ROM" + repeated DiscoveredWindow windows = 1; + int32 total_widgets = 2; + int64 generated_at_ms = 3; } ``` diff --git a/docs/z3ed/E6-z3ed-reference.md b/docs/z3ed/E6-z3ed-reference.md index 4d5c14ea..14590fd4 100644 --- a/docs/z3ed/E6-z3ed-reference.md +++ b/docs/z3ed/E6-z3ed-reference.md @@ -266,6 +266,12 @@ Options: --format Output as `table` (default) or `json` --limit Maximum widgets to display (useful for large UIs) +Each discovered widget now reports: +- Current visibility/enablement and bounding box (when available) +- Last observed frame number and UTC timestamp +- A `stale` flag when the widget hasn't appeared in the current frame +- Its underlying ImGui ID for low-level automation + Examples: # Discover all widgets currently registered z3ed agent gui discover @@ -292,10 +298,13 @@ Window: Overworld (visible) Suggested: Click button:Save State: visible, enabled Bounds: (24.0, 64.0) → (112.0, 92.0) + Last seen: frame 18432 @ 2025-01-16 19:42:05 Widget ID: 0x13fc41a2 Widgets shown: 3 of 18 (truncated) Snapshot: 2025-01-16 19:42:05 + +_Widgets that have not appeared in the current frame are marked `[STALE]` in the table output._ ``` **Use Cases**: diff --git a/docs/z3ed/QUICK_REFERENCE.md b/docs/z3ed/QUICK_REFERENCE.md index 2c830552..8ee56cb7 100644 --- a/docs/z3ed/QUICK_REFERENCE.md +++ b/docs/z3ed/QUICK_REFERENCE.md @@ -121,7 +121,7 @@ z3ed agent test results --test-id grpc_click_12345678 --include-logs z3ed agent test list --category grpc ``` -#### Widget Discovery (IT-06) 🔜 PLANNED +#### Widget Discovery (IT-06) � IN PROGRESS — telemetry available ```bash # Discover all widgets z3ed agent gui discover @@ -129,8 +129,8 @@ z3ed agent gui discover # Filter by window z3ed agent gui discover --window "Overworld" -# Get only buttons -z3ed agent gui discover --type button --format json +# Get only buttons and include hidden/disabled widgets for AI diffing +z3ed agent gui discover --type button --include-invisible --include-disabled --format json ``` #### Test Recording (IT-07) 🔜 PLANNED @@ -365,7 +365,7 @@ grpcurl ... Wait '{"condition":"window_visible:WindowName"}' # 3. Assert widget exists grpcurl ... Assert '{"condition":"exists:button:XYZ"}' -# 4. Use widget discovery (IT-06, planned) +# 4. Use widget discovery (IT-06 telemetry) z3ed agent gui discover --window "WindowName" ``` diff --git a/src/app/core/controller.cc b/src/app/core/controller.cc index 0d5b866d..6d91be00 100644 --- a/src/app/core/controller.cc +++ b/src/app/core/controller.cc @@ -7,6 +7,7 @@ #include "app/editor/editor_manager.h" #include "app/gui/background_renderer.h" #include "app/gui/theme_manager.h" +#include "app/gui/widget_id_registry.h" #include "imgui/backends/imgui_impl_sdl2.h" #include "imgui/backends/imgui_impl_sdlrenderer2.h" #include "imgui/imgui.h" @@ -65,7 +66,10 @@ absl::Status Controller::OnLoad() { ImGui::End(); #endif - RETURN_IF_ERROR(editor_manager_.Update()); + gui::WidgetIdRegistry::Instance().BeginFrame(); + absl::Status update_status = editor_manager_.Update(); + gui::WidgetIdRegistry::Instance().EndFrame(); + RETURN_IF_ERROR(update_status); return absl::OkStatus(); } diff --git a/src/app/core/proto/imgui_test_harness.proto b/src/app/core/proto/imgui_test_harness.proto index e413d317..7a2e3f67 100644 --- a/src/app/core/proto/imgui_test_harness.proto +++ b/src/app/core/proto/imgui_test_harness.proto @@ -290,6 +290,9 @@ message DiscoveredWidget { bool enabled = 7; // Currently enabled for interaction WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates uint32 widget_id = 9; // ImGui ID (debugging / direct access) + int64 last_seen_frame = 10; // Frame number when widget was last observed + int64 last_seen_at_ms = 11; // Wall-clock timestamp of last observation + bool stale = 12; // True if widget not seen in the current frame } message DiscoveredWindow { diff --git a/src/app/core/service/widget_discovery_service.cc b/src/app/core/service/widget_discovery_service.cc index f5104c4b..ec9b877e 100644 --- a/src/app/core/service/widget_discovery_service.cc +++ b/src/app/core/service/widget_discovery_service.cc @@ -1,6 +1,7 @@ #include "app/core/service/widget_discovery_service.h" #include +#include #include #include #include @@ -17,7 +18,7 @@ namespace { struct WindowEntry { int index = -1; - bool visible = true; + bool visible = false; }; } // namespace @@ -57,39 +58,17 @@ void WidgetDiscoveryService::CollectWidgets( continue; } - const std::string window_name = ExtractWindowName(path); + const std::string window_name = + info.window_name.empty() ? ExtractWindowName(path) : info.window_name; if (!MatchesWindow(window_name, window_filter_lower)) { continue; } - auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{}); - WindowEntry& entry = it->second; + const std::string label = + info.label.empty() ? ExtractLabel(path) : info.label; - if (inserted) { -#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE - if (ctx) { - ImGuiTestItemInfo window_info = - ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError); - entry.visible = (window_info.ID != 0); - } -#endif - } - - if (!include_invisible && !entry.visible) { - continue; - } - - if (entry.index == -1) { - DiscoveredWindow* window_proto = response->add_windows(); - entry.index = response->windows_size() - 1; - window_proto->set_name(window_name); - window_proto->set_visible(entry.visible); - } - - const std::string label = ExtractLabel(path); - - bool widget_enabled = true; - bool widget_visible = entry.visible; + bool widget_enabled = info.enabled; + bool widget_visible = info.visible; #if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE bool has_item_info = false; @@ -103,8 +82,18 @@ void WidgetDiscoveryService::CollectWidgets( widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0; } } +#else + (void)ctx; #endif + auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{}); + WindowEntry& entry = it->second; + if (inserted) { + entry.visible = widget_visible; + } else { + entry.visible = entry.visible || widget_visible; + } + if (!include_invisible && !widget_visible) { continue; } @@ -112,7 +101,16 @@ void WidgetDiscoveryService::CollectWidgets( continue; } + if (entry.index == -1) { + DiscoveredWindow* window_proto = response->add_windows(); + entry.index = response->windows_size() - 1; + window_proto->set_name(window_name); + window_proto->set_visible(entry.visible); + } + auto* window_proto = response->mutable_windows(entry.index); + window_proto->set_visible(entry.visible); + auto* widget_proto = window_proto->add_widgets(); widget_proto->set_path(path); widget_proto->set_label(label); @@ -126,19 +124,34 @@ void WidgetDiscoveryService::CollectWidgets( widget_proto->set_description(info.description); } - WidgetBounds* bounds = widget_proto->mutable_bounds(); + if (info.bounds.valid) { + WidgetBounds* bounds = widget_proto->mutable_bounds(); + bounds->set_min_x(info.bounds.min_x); + bounds->set_min_y(info.bounds.min_y); + bounds->set_max_x(info.bounds.max_x); + bounds->set_max_y(info.bounds.max_y); #if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE - if (ctx && has_item_info) { + } else if (ctx && has_item_info) { + WidgetBounds* bounds = widget_proto->mutable_bounds(); bounds->set_min_x(item_info.RectFull.Min.x); bounds->set_min_y(item_info.RectFull.Min.y); bounds->set_max_x(item_info.RectFull.Max.x); bounds->set_max_y(item_info.RectFull.Max.y); } else { (void)ctx; - } #else - (void)ctx; + } else { + (void)ctx; #endif + } + + widget_proto->set_last_seen_frame(info.last_seen_frame); + int64_t last_seen_ms = 0; + if (info.last_seen_time != absl::Time()) { + last_seen_ms = absl::ToUnixMillis(info.last_seen_time); + } + widget_proto->set_last_seen_at_ms(last_seen_ms); + widget_proto->set_stale(info.stale_frame_count > 0); ++total_widgets; } diff --git a/src/app/gui/widget_id_registry.cc b/src/app/gui/widget_id_registry.cc index 99fd6ba6..dfd6325f 100644 --- a/src/app/gui/widget_id_registry.cc +++ b/src/app/gui/widget_id_registry.cc @@ -1,13 +1,19 @@ #include "widget_id_registry.h" #include +#include +#include +#include #include #include +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" +#include "absl/strings/strip.h" +#include "absl/time/clock.h" #include "imgui/imgui_internal.h" // For ImGuiContext internals namespace yaze { @@ -54,11 +60,170 @@ WidgetIdRegistry& WidgetIdRegistry::Instance() { return instance; } +void WidgetIdRegistry::BeginFrame() { + ImGuiContext* ctx = ImGui::GetCurrentContext(); + if (ctx) { + current_frame_ = ctx->FrameCount; + } else if (current_frame_ >= 0) { + ++current_frame_; + } else { + current_frame_ = 0; + } + frame_time_ = absl::Now(); + for (auto& [_, info] : widgets_) { + info.seen_in_current_frame = false; + } +} + +void WidgetIdRegistry::EndFrame() { + for (auto& [_, info] : widgets_) { + if (!info.seen_in_current_frame) { + info.visible = false; + info.enabled = false; + info.bounds.valid = false; + info.stale_frame_count += 1; + } else { + info.seen_in_current_frame = false; + info.stale_frame_count = 0; + } + } + TrimStaleEntries(); +} + +namespace { + +std::string ExtractWindowFromPath(absl::string_view path) { + size_t slash = path.find('/'); + if (slash == absl::string_view::npos) { + return std::string(path); + } + return std::string(path.substr(0, slash)); +} + +std::string ExtractLabelFromPath(absl::string_view path) { + size_t colon = path.rfind(':'); + if (colon == absl::string_view::npos) { + size_t slash = path.rfind('/'); + if (slash == absl::string_view::npos) { + return std::string(path); + } + return std::string(path.substr(slash + 1)); + } + return std::string(path.substr(colon + 1)); +} + +std::string FormatTimestampUTC(const absl::Time& timestamp) { + if (timestamp == absl::Time()) { + return ""; + } + + std::chrono::system_clock::time_point chrono_time = + absl::ToChronoTime(timestamp); + std::time_t time_value = std::chrono::system_clock::to_time_t(chrono_time); + + std::tm tm_buffer; +#if defined(_WIN32) + if (gmtime_s(&tm_buffer, &time_value) != 0) { + return ""; + } +#else + if (gmtime_r(&time_value, &tm_buffer) == nullptr) { + return ""; + } +#endif + + char buffer[32]; + if (std::snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02dZ", + tm_buffer.tm_year + 1900, tm_buffer.tm_mon + 1, + tm_buffer.tm_mday, tm_buffer.tm_hour, tm_buffer.tm_min, + tm_buffer.tm_sec) <= 0) { + return ""; + } + + return std::string(buffer); +} + +WidgetIdRegistry::WidgetBounds BoundsFromImGui(const ImRect& rect) { + WidgetIdRegistry::WidgetBounds bounds; + bounds.min_x = rect.Min.x; + bounds.min_y = rect.Min.y; + bounds.max_x = rect.Max.x; + bounds.max_y = rect.Max.y; + bounds.valid = true; + return bounds; +} + +} // namespace + void WidgetIdRegistry::RegisterWidget(const std::string& full_path, - const std::string& type, ImGuiID imgui_id, - const std::string& description) { - WidgetInfo info{full_path, type, imgui_id, description}; - widgets_[full_path] = info; + const std::string& type, + ImGuiID imgui_id, + const std::string& description, + const WidgetMetadata& metadata) { + WidgetInfo& info = widgets_[full_path]; + info.full_path = full_path; + info.type = type; + info.imgui_id = imgui_id; + info.description = description; + + if (metadata.label.has_value()) { + info.label = NormalizeLabel(*metadata.label); + } else { + info.label = NormalizeLabel(ExtractLabelFromPath(full_path)); + } + if (info.label.empty()) { + info.label = ExtractLabelFromPath(full_path); + } + + if (metadata.window_name.has_value()) { + info.window_name = NormalizeLabel(*metadata.window_name); + } else { + info.window_name = NormalizePathSegment(ExtractWindowFromPath(full_path)); + } + if (info.window_name.empty()) { + info.window_name = ExtractWindowFromPath(full_path); + } + + ImGuiContext* ctx = ImGui::GetCurrentContext(); + absl::Time observed_at = absl::Now(); + + if (ctx) { + const ImGuiLastItemData& last = ctx->LastItemData; + if (metadata.visible.has_value()) { + info.visible = *metadata.visible; + } else { + info.visible = (last.StatusFlags & ImGuiItemStatusFlags_Visible) != 0; + } + + if (metadata.enabled.has_value()) { + info.enabled = *metadata.enabled; + } else { + info.enabled = (last.ItemFlags & ImGuiItemFlags_Disabled) == 0; + } + + if (metadata.bounds.has_value()) { + info.bounds = *metadata.bounds; + } else { + info.bounds = BoundsFromImGui(last.Rect); + } + + info.last_seen_frame = ctx->FrameCount; + } else { + info.visible = metadata.visible.value_or(true); + info.enabled = metadata.enabled.value_or(true); + if (metadata.bounds.has_value()) { + info.bounds = *metadata.bounds; + } else { + info.bounds.valid = false; + } + if (current_frame_ >= 0) { + info.last_seen_frame = current_frame_; + } + } + + info.last_seen_time = observed_at; + info.seen_in_current_frame = true; + info.stale_frame_count = 0; } std::vector WidgetIdRegistry::FindWidgets( @@ -113,7 +278,10 @@ const WidgetIdRegistry::WidgetInfo* WidgetIdRegistry::GetWidgetInfo( return nullptr; } -void WidgetIdRegistry::Clear() { widgets_.clear(); } +void WidgetIdRegistry::Clear() { + widgets_.clear(); + current_frame_ = -1; +} std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const { std::ostringstream ss; @@ -130,12 +298,36 @@ std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const { ss << " {\n"; ss << absl::StrFormat(" \"path\": \"%s\",\n", path); ss << absl::StrFormat(" \"type\": \"%s\",\n", info.type); - ss << absl::StrFormat(" \"imgui_id\": %u", info.imgui_id); + ss << absl::StrFormat(" \"imgui_id\": %u,\n", info.imgui_id); + ss << absl::StrFormat(" \"label\": \"%s\",\n", info.label); + ss << absl::StrFormat(" \"window\": \"%s\",\n", info.window_name); + ss << absl::StrFormat(" \"visible\": %s,\n", + info.visible ? "true" : "false"); + ss << absl::StrFormat(" \"enabled\": %s,\n", + info.enabled ? "true" : "false"); + if (info.bounds.valid) { + ss << absl::StrFormat( + " \"bounds\": {\"min\": [%0.1f, %0.1f], \"max\": [%0.1f, %0.1f]},\n", + info.bounds.min_x, info.bounds.min_y, info.bounds.max_x, + info.bounds.max_y); + } else { + ss << " \"bounds\": null,\n"; + } + ss << absl::StrFormat(" \"last_seen_frame\": %d,\n", + info.last_seen_frame); + std::string iso_timestamp = FormatTimestampUTC(info.last_seen_time); + ss << absl::StrFormat(" \"last_seen_at\": \"%s\",\n", + iso_timestamp); + ss << absl::StrFormat(" \"stale\": %s", + info.stale_frame_count > 0 ? "true" : "false"); if (!info.description.empty()) { ss << ",\n"; - ss << absl::StrFormat(" \"description\": \"%s\"", info.description); + ss << absl::StrFormat(" \"description\": \"%s\"\n", + info.description); + } else { + ss << "\n"; } - ss << "\n }"; + ss << " }"; } ss << "\n ]\n"; @@ -148,6 +340,22 @@ std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const { ss << absl::StrFormat(" - path: \"%s\"\n", path); ss << absl::StrFormat(" type: %s\n", info.type); ss << absl::StrFormat(" imgui_id: %u\n", info.imgui_id); + ss << absl::StrFormat(" label: \"%s\"\n", info.label); + ss << absl::StrFormat(" window: \"%s\"\n", info.window_name); + ss << absl::StrFormat(" visible: %s\n", info.visible ? "true" : "false"); + ss << absl::StrFormat(" enabled: %s\n", info.enabled ? "true" : "false"); + if (info.bounds.valid) { + ss << " bounds:\n"; + ss << absl::StrFormat(" min: [%0.1f, %0.1f]\n", info.bounds.min_x, + info.bounds.min_y); + ss << absl::StrFormat(" max: [%0.1f, %0.1f]\n", info.bounds.max_x, + info.bounds.max_y); + } + ss << absl::StrFormat(" last_seen_frame: %d\n", info.last_seen_frame); + std::string iso_timestamp = FormatTimestampUTC(info.last_seen_time); + ss << absl::StrFormat(" last_seen_at: %s\n", iso_timestamp); + ss << absl::StrFormat(" stale: %s\n", + info.stale_frame_count > 0 ? "true" : "false"); // Parse hierarchical context from path std::vector segments = absl::StrSplit(path, '/'); @@ -200,5 +408,39 @@ void WidgetIdRegistry::ExportCatalogToFile(const std::string& output_file, } } +std::string WidgetIdRegistry::NormalizeLabel(absl::string_view label) { + size_t pos = label.find("##"); + if (pos != absl::string_view::npos) { + label = label.substr(0, pos); + } + std::string sanitized = std::string(absl::StripAsciiWhitespace(label)); + return sanitized; +} + +std::string WidgetIdRegistry::NormalizePathSegment(absl::string_view segment) { + return NormalizeLabel(segment); +} + +void WidgetIdRegistry::TrimStaleEntries() { + auto it = widgets_.begin(); + while (it != widgets_.end()) { + if (ShouldPrune(it->second)) { + it = widgets_.erase(it); + } else { + ++it; + } + } +} + +bool WidgetIdRegistry::ShouldPrune(const WidgetInfo& info) const { + if (info.last_seen_frame < 0 || stale_frame_limit_ <= 0) { + return false; + } + if (current_frame_ < 0) { + return false; + } + return (current_frame_ - info.last_seen_frame) > stale_frame_limit_; +} + } // namespace gui } // namespace yaze diff --git a/src/app/gui/widget_id_registry.h b/src/app/gui/widget_id_registry.h index 7a016b74..a0db68ad 100644 --- a/src/app/gui/widget_id_registry.h +++ b/src/app/gui/widget_id_registry.h @@ -1,10 +1,14 @@ #ifndef YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_ #define YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_ +#include +#include #include #include #include +#include "absl/strings/string_view.h" +#include "absl/time/time.h" #include "imgui/imgui.h" namespace yaze { @@ -62,20 +66,50 @@ class WidgetIdScope { */ class WidgetIdRegistry { public: + struct WidgetBounds { + float min_x = 0.0f; + float min_y = 0.0f; + float max_x = 0.0f; + float max_y = 0.0f; + bool valid = false; + }; + + struct WidgetMetadata { + std::optional label; + std::optional window_name; + std::optional visible; + std::optional enabled; + std::optional bounds; + }; + struct WidgetInfo { std::string full_path; // e.g. "Overworld/Canvas/canvas:Map" std::string type; // e.g. "button", "input", "canvas", "table" ImGuiID imgui_id; // ImGui's internal ID std::string description; // Optional human-readable description + std::string label; // Sanitized display label (without IDs/icons) + std::string window_name; // Window this widget was last seen in + bool visible = true; // Visibility in the most recent frame + bool enabled = true; // Enabled state in the most recent frame + WidgetBounds bounds; // Bounding box in screen space + int last_seen_frame = -1; + absl::Time last_seen_time; + bool seen_in_current_frame = false; + int stale_frame_count = 0; }; static WidgetIdRegistry& Instance(); + // Frame lifecycle - call once per ImGui frame + void BeginFrame(); + void EndFrame(); + // Register a widget for discovery // Should be called after widget is created (when ImGui::GetItemID() is valid) void RegisterWidget(const std::string& full_path, const std::string& type, ImGuiID imgui_id, - const std::string& description = ""); + const std::string& description = "", + const WidgetMetadata& metadata = WidgetMetadata()); // Query widgets for test automation std::vector FindWidgets(const std::string& pattern) const; @@ -96,9 +130,19 @@ class WidgetIdRegistry { void ExportCatalogToFile(const std::string& output_file, const std::string& format = "yaml") const; + // Helper utilities for consistent naming + static std::string NormalizeLabel(absl::string_view label); + static std::string NormalizePathSegment(absl::string_view segment); + private: WidgetIdRegistry() = default; + void TrimStaleEntries(); + bool ShouldPrune(const WidgetInfo& info) const; + std::unordered_map widgets_; + int current_frame_ = -1; + absl::Time frame_time_; + int stale_frame_limit_ = 600; // frames before pruning a widget }; // RAII helper macros for convenient scoping diff --git a/src/cli/handlers/agent/gui_commands.cc b/src/cli/handlers/agent/gui_commands.cc index cc7f788a..a6a04839 100644 --- a/src/cli/handlers/agent/gui_commands.cc +++ b/src/cli/handlers/agent/gui_commands.cc @@ -227,11 +227,31 @@ absl::Status HandleGuiDiscoverCommand(const std::vector& arg_vec) { << (widget.visible ? "true" : "false") << ",\n"; std::cout << " \"enabled\": " << (widget.enabled ? "true" : "false") << ",\n"; - std::cout << " \"bounds\": { \"min\": [" << widget.bounds.min_x - << ", " << widget.bounds.min_y << "], \"max\": [" - << widget.bounds.max_x << ", " << widget.bounds.max_y - << "] },\n"; - std::cout << " \"widgetId\": " << widget.widget_id << "\n"; + if (widget.has_bounds) { + std::cout << " \"bounds\": { \"min\": [" + << widget.bounds.min_x << ", " << widget.bounds.min_y + << "], \"max\": [" << widget.bounds.max_x << ", " + << widget.bounds.max_y << "] },\n"; + } else { + std::cout << " \"bounds\": null,\n"; + } + std::cout << " \"widgetId\": " << widget.widget_id << ",\n"; + std::cout << " \"lastSeenFrame\": " + << widget.last_seen_frame << ",\n"; + std::cout << " \"lastSeenAt\": "; + if (widget.last_seen_at.has_value()) { + std::cout + << "\"" + << JsonEscape(absl::FormatTime("%Y-%m-%dT%H:%M:%SZ", + *widget.last_seen_at, + absl::UTCTimeZone())) + << "\""; + } else { + std::cout << "null"; + } + std::cout << ",\n"; + std::cout << " \"stale\": " + << (widget.stale ? "true" : "false") << "\n"; std::cout << " }"; if (i + 1 < window.widgets.size()) { std::cout << ","; @@ -284,9 +304,24 @@ absl::Status HandleGuiDiscoverCommand(const std::vector& arg_vec) { std::cout << " Suggested: " << widget.suggested_action << "\n"; std::cout << " State: " << (widget.visible ? "visible" : "hidden") << ", " << (widget.enabled ? "enabled" : "disabled") << "\n"; - std::cout << absl::StrFormat(" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n", - widget.bounds.min_x, widget.bounds.min_y, - widget.bounds.max_x, widget.bounds.max_y); + if (widget.has_bounds) { + std::cout << absl::StrFormat( + " Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n", widget.bounds.min_x, + widget.bounds.min_y, widget.bounds.max_x, widget.bounds.max_y); + } else { + std::cout << " Bounds: (not available)\n"; + } + std::cout << " Last seen: frame " << widget.last_seen_frame; + if (widget.last_seen_at.has_value()) { + std::cout << " @ " + << absl::FormatTime("%Y-%m-%d %H:%M:%S", + *widget.last_seen_at, + absl::LocalTimeZone()); + } + if (widget.stale) { + std::cout << " [STALE]"; + } + std::cout << "\n"; std::cout << " Widget ID: 0x" << std::hex << widget.widget_id << std::dec << "\n"; } diff --git a/src/cli/service/gui_automation_client.cc b/src/cli/service/gui_automation_client.cc index 39962cda..4e64edd0 100644 --- a/src/cli/service/gui_automation_client.cc +++ b/src/cli/service/gui_automation_client.cc @@ -519,11 +519,19 @@ absl::StatusOr GuiAutomationClient::DiscoverWidgets( widget.suggested_action = widget_proto.suggested_action(); widget.visible = widget_proto.visible(); widget.enabled = widget_proto.enabled(); - widget.bounds.min_x = widget_proto.bounds().min_x(); - widget.bounds.min_y = widget_proto.bounds().min_y(); - widget.bounds.max_x = widget_proto.bounds().max_x(); - widget.bounds.max_y = widget_proto.bounds().max_y(); + widget.has_bounds = widget_proto.has_bounds(); + if (widget.has_bounds) { + widget.bounds.min_x = widget_proto.bounds().min_x(); + widget.bounds.min_y = widget_proto.bounds().min_y(); + widget.bounds.max_x = widget_proto.bounds().max_x(); + widget.bounds.max_y = widget_proto.bounds().max_y(); + } else { + widget.bounds = WidgetBoundingBox(); + } widget.widget_id = widget_proto.widget_id(); + widget.last_seen_frame = widget_proto.last_seen_frame(); + widget.last_seen_at = OptionalTimeFromMillis(widget_proto.last_seen_at_ms()); + widget.stale = widget_proto.stale(); window_info.widgets.push_back(std::move(widget)); } diff --git a/src/cli/service/gui_automation_client.h b/src/cli/service/gui_automation_client.h index 4f3ae678..f6cd3c4a 100644 --- a/src/cli/service/gui_automation_client.h +++ b/src/cli/service/gui_automation_client.h @@ -152,6 +152,10 @@ struct WidgetDescriptor { bool enabled = true; WidgetBoundingBox bounds; uint32_t widget_id = 0; + bool has_bounds = false; + int64_t last_seen_frame = -1; + std::optional last_seen_at; + bool stale = false; }; struct DiscoveredWindowInfo {