scawful/yaze
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance widget discovery with telemetry data and improve output formatting

Browse Source
This commit is contained in:
scawful
2025-10-02 22:50:47 -04:00
parent 21074f6445
commit c202aa9fa4
11 changed files with 457 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
docs/z3ed/E6-z3ed-implementation-plan.md
View File

@@ -202,31 +202,48 @@ z3ed gui discover --window "Overworld"
z3ed gui discover --type button
```
**API Schema**:
**API Schema (current)**:
```proto
message DiscoverWidgetsRequest {
string window_filter = 1; // Optional: filter by window name
enum WidgetType { ALL = 0; BUTTON = 1; INPUT = 2; MENU = 3; TAB = 4; CHECKBOX = 5; }
string window_filter = 1;
WidgetType type_filter = 2;
string path_prefix = 3;
bool include_invisible = 4;
bool include_disabled = 5;
}
message WidgetBounds {
float min_x = 1;
float min_y = 2;
float max_x = 3;
float max_y = 4;
}
message DiscoveredWidget {
string path = 1;
string label = 2;
string type = 3;
string description = 4;
string suggested_action = 5;
bool visible = 6;
bool enabled = 7;
WidgetBounds bounds = 8;
uint32 widget_id = 9;
int64 last_seen_frame = 10;
int64 last_seen_at_ms = 11;
bool stale = 12;
}
message DiscoveredWindow {
string name = 1;
bool visible = 2;
repeated DiscoveredWidget widgets = 3;
}
message DiscoverWidgetsResponse {
repeated WindowInfo windows = 1;
}
message WindowInfo {
string name = 1;
bool is_visible = 2;
repeated WidgetInfo widgets = 3;
}
message WidgetInfo {
string id = 1;
string label = 2;
string type = 3; // "button", "input", "menu", etc.
bool is_enabled = 4;
string position = 5; // "x,y,width,height"
string suggested_action = 6; // "Click button:Open ROM"
repeated DiscoveredWindow windows = 1;
int32 total_widgets = 2;
int64 generated_at_ms = 3;
}
```

9
docs/z3ed/E6-z3ed-reference.md
View File

@@ -266,6 +266,12 @@ Options:
--format <mode> Output as `table` (default) or `json`
--limit <n> Maximum widgets to display (useful for large UIs)
Each discovered widget now reports:
- Current visibility/enablement and bounding box (when available)
- Last observed frame number and UTC timestamp
- A `stale` flag when the widget hasn't appeared in the current frame
- Its underlying ImGui ID for low-level automation
Examples:
# Discover all widgets currently registered
z3ed agent gui discover
@@ -292,10 +298,13 @@ Window: Overworld (visible)
Suggested: Click button:Save
State: visible, enabled
Bounds: (24.0, 64.0) → (112.0, 92.0)
Last seen: frame 18432 @ 2025-01-16 19:42:05
Widget ID: 0x13fc41a2
Widgets shown: 3 of 18 (truncated)
Snapshot: 2025-01-16 19:42:05
_Widgets that have not appeared in the current frame are marked `[STALE]` in the table output._
```
**Use Cases**:

8
docs/z3ed/QUICK_REFERENCE.md
View File

@@ -121,7 +121,7 @@ z3ed agent test results --test-id grpc_click_12345678 --include-logs
z3ed agent test list --category grpc
```
#### Widget Discovery (IT-06) 🔜 PLANNED
#### Widget Discovery (IT-06) <EFBFBD> IN PROGRESS — telemetry available
```bash
# Discover all widgets
z3ed agent gui discover
@@ -129,8 +129,8 @@ z3ed agent gui discover
# Filter by window
z3ed agent gui discover --window "Overworld"
# Get only buttons
z3ed agent gui discover --type button --format json
# Get only buttons and include hidden/disabled widgets for AI diffing
z3ed agent gui discover --type button --include-invisible --include-disabled --format json
```
#### Test Recording (IT-07) 🔜 PLANNED
@@ -365,7 +365,7 @@ grpcurl ... Wait '{"condition":"window_visible:WindowName"}'
# 3. Assert widget exists
grpcurl ... Assert '{"condition":"exists:button:XYZ"}'
# 4. Use widget discovery (IT-06, planned)
# 4. Use widget discovery (IT-06 telemetry)
z3ed agent gui discover --window "WindowName"
```

6
src/app/core/controller.cc
View File

@@ -7,6 +7,7 @@
#include "app/editor/editor_manager.h"
#include "app/gui/background_renderer.h"
#include "app/gui/theme_manager.h"
#include "app/gui/widget_id_registry.h"
#include "imgui/backends/imgui_impl_sdl2.h"
#include "imgui/backends/imgui_impl_sdlrenderer2.h"
#include "imgui/imgui.h"
@@ -65,7 +66,10 @@ absl::Status Controller::OnLoad() {
ImGui::End();
#endif
RETURN_IF_ERROR(editor_manager_.Update());
gui::WidgetIdRegistry::Instance().BeginFrame();
absl::Status update_status = editor_manager_.Update();
gui::WidgetIdRegistry::Instance().EndFrame();
RETURN_IF_ERROR(update_status);
return absl::OkStatus();
}

3
src/app/core/proto/imgui_test_harness.proto
View File

@@ -290,6 +290,9 @@ message DiscoveredWidget {
bool enabled = 7; // Currently enabled for interaction
WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates
uint32 widget_id = 9; // ImGui ID (debugging / direct access)
int64 last_seen_frame = 10; // Frame number when widget was last observed
int64 last_seen_at_ms = 11; // Wall-clock timestamp of last observation
bool stale = 12; // True if widget not seen in the current frame
}
message DiscoveredWindow {

79
src/app/core/service/widget_discovery_service.cc
View File

@@ -1,6 +1,7 @@
#include "app/core/service/widget_discovery_service.h"
#include <algorithm>
#include <cstdint>
#include <map>
#include <string>
#include <utility>
@@ -17,7 +18,7 @@ namespace {
struct WindowEntry {
int index = -1;
bool visible = true;
bool visible = false;
};
} // namespace
@@ -57,39 +58,17 @@ void WidgetDiscoveryService::CollectWidgets(
continue;
}
const std::string window_name = ExtractWindowName(path);
const std::string window_name =
info.window_name.empty() ? ExtractWindowName(path) : info.window_name;
if (!MatchesWindow(window_name, window_filter_lower)) {
continue;
}
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
WindowEntry& entry = it->second;
const std::string label =
info.label.empty() ? ExtractLabel(path) : info.label;
if (inserted) {
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
if (ctx) {
ImGuiTestItemInfo window_info =
ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError);
entry.visible = (window_info.ID != 0);
}
#endif
}
if (!include_invisible && !entry.visible) {
continue;
}
if (entry.index == -1) {
DiscoveredWindow* window_proto = response->add_windows();
entry.index = response->windows_size() - 1;
window_proto->set_name(window_name);
window_proto->set_visible(entry.visible);
}
const std::string label = ExtractLabel(path);
bool widget_enabled = true;
bool widget_visible = entry.visible;
bool widget_enabled = info.enabled;
bool widget_visible = info.visible;
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
bool has_item_info = false;
@@ -103,8 +82,18 @@ void WidgetDiscoveryService::CollectWidgets(
widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0;
}
}
#else
(void)ctx;
#endif
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
WindowEntry& entry = it->second;
if (inserted) {
entry.visible = widget_visible;
} else {
entry.visible = entry.visible || widget_visible;
}
if (!include_invisible && !widget_visible) {
continue;
}
@@ -112,7 +101,16 @@ void WidgetDiscoveryService::CollectWidgets(
continue;
}
if (entry.index == -1) {
DiscoveredWindow* window_proto = response->add_windows();
entry.index = response->windows_size() - 1;
window_proto->set_name(window_name);
window_proto->set_visible(entry.visible);
}
auto* window_proto = response->mutable_windows(entry.index);
window_proto->set_visible(entry.visible);
auto* widget_proto = window_proto->add_widgets();
widget_proto->set_path(path);
widget_proto->set_label(label);
@@ -126,19 +124,34 @@ void WidgetDiscoveryService::CollectWidgets(
widget_proto->set_description(info.description);
}
WidgetBounds* bounds = widget_proto->mutable_bounds();
if (info.bounds.valid) {
WidgetBounds* bounds = widget_proto->mutable_bounds();
bounds->set_min_x(info.bounds.min_x);
bounds->set_min_y(info.bounds.min_y);
bounds->set_max_x(info.bounds.max_x);
bounds->set_max_y(info.bounds.max_y);
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
if (ctx && has_item_info) {
} else if (ctx && has_item_info) {
WidgetBounds* bounds = widget_proto->mutable_bounds();
bounds->set_min_x(item_info.RectFull.Min.x);
bounds->set_min_y(item_info.RectFull.Min.y);
bounds->set_max_x(item_info.RectFull.Max.x);
bounds->set_max_y(item_info.RectFull.Max.y);
} else {
(void)ctx;
}
#else
(void)ctx;
} else {
(void)ctx;
#endif
}
widget_proto->set_last_seen_frame(info.last_seen_frame);
int64_t last_seen_ms = 0;
if (info.last_seen_time != absl::Time()) {
last_seen_ms = absl::ToUnixMillis(info.last_seen_time);
}
widget_proto->set_last_seen_at_ms(last_seen_ms);
widget_proto->set_stale(info.stale_frame_count > 0);
++total_widgets;
}

258
src/app/gui/widget_id_registry.cc
View File

@@ -1,13 +1,19 @@
#include "widget_id_registry.h"
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <ctime>
#include <fstream>
#include <sstream>
#include "absl/strings/ascii.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_split.h"
#include "absl/strings/strip.h"
#include "absl/time/clock.h"
#include "imgui/imgui_internal.h" // For ImGuiContext internals
namespace yaze {
@@ -54,11 +60,170 @@ WidgetIdRegistry& WidgetIdRegistry::Instance() {
return instance;
}
void WidgetIdRegistry::BeginFrame() {
ImGuiContext* ctx = ImGui::GetCurrentContext();
if (ctx) {
current_frame_ = ctx->FrameCount;
} else if (current_frame_ >= 0) {
++current_frame_;
} else {
current_frame_ = 0;
}
frame_time_ = absl::Now();
for (auto& [_, info] : widgets_) {
info.seen_in_current_frame = false;
}
}
void WidgetIdRegistry::EndFrame() {
for (auto& [_, info] : widgets_) {
if (!info.seen_in_current_frame) {
info.visible = false;
info.enabled = false;
info.bounds.valid = false;
info.stale_frame_count += 1;
} else {
info.seen_in_current_frame = false;
info.stale_frame_count = 0;
}
}
TrimStaleEntries();
}
namespace {
std::string ExtractWindowFromPath(absl::string_view path) {
size_t slash = path.find('/');
if (slash == absl::string_view::npos) {
return std::string(path);
}
return std::string(path.substr(0, slash));
}
std::string ExtractLabelFromPath(absl::string_view path) {
size_t colon = path.rfind(':');
if (colon == absl::string_view::npos) {
size_t slash = path.rfind('/');
if (slash == absl::string_view::npos) {
return std::string(path);
}
return std::string(path.substr(slash + 1));
}
return std::string(path.substr(colon + 1));
}
std::string FormatTimestampUTC(const absl::Time& timestamp) {
if (timestamp == absl::Time()) {
return "";
}
std::chrono::system_clock::time_point chrono_time =
absl::ToChronoTime(timestamp);
std::time_t time_value = std::chrono::system_clock::to_time_t(chrono_time);
std::tm tm_buffer;
#if defined(_WIN32)
if (gmtime_s(&tm_buffer, &time_value) != 0) {
return "";
}
#else
if (gmtime_r(&time_value, &tm_buffer) == nullptr) {
return "";
}
#endif
char buffer[32];
if (std::snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02dZ",
tm_buffer.tm_year + 1900, tm_buffer.tm_mon + 1,
tm_buffer.tm_mday, tm_buffer.tm_hour, tm_buffer.tm_min,
tm_buffer.tm_sec) <= 0) {
return "";
}
return std::string(buffer);
}
WidgetIdRegistry::WidgetBounds BoundsFromImGui(const ImRect& rect) {
WidgetIdRegistry::WidgetBounds bounds;
bounds.min_x = rect.Min.x;
bounds.min_y = rect.Min.y;
bounds.max_x = rect.Max.x;
bounds.max_y = rect.Max.y;
bounds.valid = true;
return bounds;
}
} // namespace
void WidgetIdRegistry::RegisterWidget(const std::string& full_path,
const std::string& type, ImGuiID imgui_id,
const std::string& description) {
WidgetInfo info{full_path, type, imgui_id, description};
widgets_[full_path] = info;
const std::string& type,
ImGuiID imgui_id,
const std::string& description,
const WidgetMetadata& metadata) {
WidgetInfo& info = widgets_[full_path];
info.full_path = full_path;
info.type = type;
info.imgui_id = imgui_id;
info.description = description;
if (metadata.label.has_value()) {
info.label = NormalizeLabel(*metadata.label);
} else {
info.label = NormalizeLabel(ExtractLabelFromPath(full_path));
}
if (info.label.empty()) {
info.label = ExtractLabelFromPath(full_path);
}
if (metadata.window_name.has_value()) {
info.window_name = NormalizeLabel(*metadata.window_name);
} else {
info.window_name = NormalizePathSegment(ExtractWindowFromPath(full_path));
}
if (info.window_name.empty()) {
info.window_name = ExtractWindowFromPath(full_path);
}
ImGuiContext* ctx = ImGui::GetCurrentContext();
absl::Time observed_at = absl::Now();
if (ctx) {
const ImGuiLastItemData& last = ctx->LastItemData;
if (metadata.visible.has_value()) {
info.visible = *metadata.visible;
} else {
info.visible = (last.StatusFlags & ImGuiItemStatusFlags_Visible) != 0;
}
if (metadata.enabled.has_value()) {
info.enabled = *metadata.enabled;
} else {
info.enabled = (last.ItemFlags & ImGuiItemFlags_Disabled) == 0;
}
if (metadata.bounds.has_value()) {
info.bounds = *metadata.bounds;
} else {
info.bounds = BoundsFromImGui(last.Rect);
}
info.last_seen_frame = ctx->FrameCount;
} else {
info.visible = metadata.visible.value_or(true);
info.enabled = metadata.enabled.value_or(true);
if (metadata.bounds.has_value()) {
info.bounds = *metadata.bounds;
} else {
info.bounds.valid = false;
}
if (current_frame_ >= 0) {
info.last_seen_frame = current_frame_;
}
}
info.last_seen_time = observed_at;
info.seen_in_current_frame = true;
info.stale_frame_count = 0;
}
std::vector<std::string> WidgetIdRegistry::FindWidgets(
@@ -113,7 +278,10 @@ const WidgetIdRegistry::WidgetInfo* WidgetIdRegistry::GetWidgetInfo(
return nullptr;
}
void WidgetIdRegistry::Clear() { widgets_.clear(); }
void WidgetIdRegistry::Clear() {
widgets_.clear();
current_frame_ = -1;
}
std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const {
std::ostringstream ss;
@@ -130,12 +298,36 @@ std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const {
ss << " {\n";
ss << absl::StrFormat(" \"path\": \"%s\",\n", path);
ss << absl::StrFormat(" \"type\": \"%s\",\n", info.type);
ss << absl::StrFormat(" \"imgui_id\": %u", info.imgui_id);
ss << absl::StrFormat(" \"imgui_id\": %u,\n", info.imgui_id);
ss << absl::StrFormat(" \"label\": \"%s\",\n", info.label);
ss << absl::StrFormat(" \"window\": \"%s\",\n", info.window_name);
ss << absl::StrFormat(" \"visible\": %s,\n",
info.visible ? "true" : "false");
ss << absl::StrFormat(" \"enabled\": %s,\n",
info.enabled ? "true" : "false");
if (info.bounds.valid) {
ss << absl::StrFormat(
" \"bounds\": {\"min\": [%0.1f, %0.1f], \"max\": [%0.1f, %0.1f]},\n",
info.bounds.min_x, info.bounds.min_y, info.bounds.max_x,
info.bounds.max_y);
} else {
ss << " \"bounds\": null,\n";
}
ss << absl::StrFormat(" \"last_seen_frame\": %d,\n",
info.last_seen_frame);
std::string iso_timestamp = FormatTimestampUTC(info.last_seen_time);
ss << absl::StrFormat(" \"last_seen_at\": \"%s\",\n",
iso_timestamp);
ss << absl::StrFormat(" \"stale\": %s",
info.stale_frame_count > 0 ? "true" : "false");
if (!info.description.empty()) {
ss << ",\n";
ss << absl::StrFormat(" \"description\": \"%s\"", info.description);
ss << absl::StrFormat(" \"description\": \"%s\"\n",
info.description);
} else {
ss << "\n";
}
ss << "\n }";
ss << " }";
}
ss << "\n ]\n";
@@ -148,6 +340,22 @@ std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const {
ss << absl::StrFormat(" - path: \"%s\"\n", path);
ss << absl::StrFormat(" type: %s\n", info.type);
ss << absl::StrFormat(" imgui_id: %u\n", info.imgui_id);
ss << absl::StrFormat(" label: \"%s\"\n", info.label);
ss << absl::StrFormat(" window: \"%s\"\n", info.window_name);
ss << absl::StrFormat(" visible: %s\n", info.visible ? "true" : "false");
ss << absl::StrFormat(" enabled: %s\n", info.enabled ? "true" : "false");
if (info.bounds.valid) {
ss << " bounds:\n";
ss << absl::StrFormat(" min: [%0.1f, %0.1f]\n", info.bounds.min_x,
info.bounds.min_y);
ss << absl::StrFormat(" max: [%0.1f, %0.1f]\n", info.bounds.max_x,
info.bounds.max_y);
}
ss << absl::StrFormat(" last_seen_frame: %d\n", info.last_seen_frame);
std::string iso_timestamp = FormatTimestampUTC(info.last_seen_time);
ss << absl::StrFormat(" last_seen_at: %s\n", iso_timestamp);
ss << absl::StrFormat(" stale: %s\n",
info.stale_frame_count > 0 ? "true" : "false");
// Parse hierarchical context from path
std::vector<std::string> segments = absl::StrSplit(path, '/');
@@ -200,5 +408,39 @@ void WidgetIdRegistry::ExportCatalogToFile(const std::string& output_file,
}
}
std::string WidgetIdRegistry::NormalizeLabel(absl::string_view label) {
size_t pos = label.find("##");
if (pos != absl::string_view::npos) {
label = label.substr(0, pos);
}
std::string sanitized = std::string(absl::StripAsciiWhitespace(label));
return sanitized;
}
std::string WidgetIdRegistry::NormalizePathSegment(absl::string_view segment) {
return NormalizeLabel(segment);
}
void WidgetIdRegistry::TrimStaleEntries() {
auto it = widgets_.begin();
while (it != widgets_.end()) {
if (ShouldPrune(it->second)) {
it = widgets_.erase(it);
} else {
++it;
}
}
}
bool WidgetIdRegistry::ShouldPrune(const WidgetInfo& info) const {
if (info.last_seen_frame < 0 || stale_frame_limit_ <= 0) {
return false;
}
if (current_frame_ < 0) {
return false;
}
return (current_frame_ - info.last_seen_frame) > stale_frame_limit_;
}
} // namespace gui
} // namespace yaze

46
src/app/gui/widget_id_registry.h
View File

@@ -1,10 +1,14 @@
#ifndef YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_
#define YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_
#include <cstdint>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "imgui/imgui.h"
namespace yaze {
@@ -62,20 +66,50 @@ class WidgetIdScope {
*/
class WidgetIdRegistry {
public:
struct WidgetBounds {
float min_x = 0.0f;
float min_y = 0.0f;
float max_x = 0.0f;
float max_y = 0.0f;
bool valid = false;
};
struct WidgetMetadata {
std::optional<std::string> label;
std::optional<std::string> window_name;
std::optional<bool> visible;
std::optional<bool> enabled;
std::optional<WidgetBounds> bounds;
};
struct WidgetInfo {
std::string full_path; // e.g. "Overworld/Canvas/canvas:Map"
std::string type; // e.g. "button", "input", "canvas", "table"
ImGuiID imgui_id; // ImGui's internal ID
std::string description; // Optional human-readable description
std::string label; // Sanitized display label (without IDs/icons)
std::string window_name; // Window this widget was last seen in
bool visible = true; // Visibility in the most recent frame
bool enabled = true; // Enabled state in the most recent frame
WidgetBounds bounds; // Bounding box in screen space
int last_seen_frame = -1;
absl::Time last_seen_time;
bool seen_in_current_frame = false;
int stale_frame_count = 0;
};
static WidgetIdRegistry& Instance();
// Frame lifecycle - call once per ImGui frame
void BeginFrame();
void EndFrame();
// Register a widget for discovery
// Should be called after widget is created (when ImGui::GetItemID() is valid)
void RegisterWidget(const std::string& full_path, const std::string& type,
ImGuiID imgui_id,
const std::string& description = "");
const std::string& description = "",
const WidgetMetadata& metadata = WidgetMetadata());
// Query widgets for test automation
std::vector<std::string> FindWidgets(const std::string& pattern) const;
@@ -96,9 +130,19 @@ class WidgetIdRegistry {
void ExportCatalogToFile(const std::string& output_file,
const std::string& format = "yaml") const;
// Helper utilities for consistent naming
static std::string NormalizeLabel(absl::string_view label);
static std::string NormalizePathSegment(absl::string_view segment);
private:
WidgetIdRegistry() = default;
void TrimStaleEntries();
bool ShouldPrune(const WidgetInfo& info) const;
std::unordered_map<std::string, WidgetInfo> widgets_;
int current_frame_ = -1;
absl::Time frame_time_;
int stale_frame_limit_ = 600; // frames before pruning a widget
};
// RAII helper macros for convenient scoping

51
src/cli/handlers/agent/gui_commands.cc
View File

@@ -227,11 +227,31 @@ absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
<< (widget.visible ? "true" : "false") << ",\n";
std::cout << " \"enabled\": "
<< (widget.enabled ? "true" : "false") << ",\n";
std::cout << " \"bounds\": { \"min\": [" << widget.bounds.min_x
<< ", " << widget.bounds.min_y << "], \"max\": ["
<< widget.bounds.max_x << ", " << widget.bounds.max_y
<< "] },\n";
std::cout << " \"widgetId\": " << widget.widget_id << "\n";
if (widget.has_bounds) {
std::cout << " \"bounds\": { \"min\": ["
<< widget.bounds.min_x << ", " << widget.bounds.min_y
<< "], \"max\": [" << widget.bounds.max_x << ", "
<< widget.bounds.max_y << "] },\n";
} else {
std::cout << " \"bounds\": null,\n";
}
std::cout << " \"widgetId\": " << widget.widget_id << ",\n";
std::cout << " \"lastSeenFrame\": "
<< widget.last_seen_frame << ",\n";
std::cout << " \"lastSeenAt\": ";
if (widget.last_seen_at.has_value()) {
std::cout
<< "\""
<< JsonEscape(absl::FormatTime("%Y-%m-%dT%H:%M:%SZ",
*widget.last_seen_at,
absl::UTCTimeZone()))
<< "\"";
} else {
std::cout << "null";
}
std::cout << ",\n";
std::cout << " \"stale\": "
<< (widget.stale ? "true" : "false") << "\n";
std::cout << " }";
if (i + 1 < window.widgets.size()) {
std::cout << ",";
@@ -284,9 +304,24 @@ absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
std::cout << " Suggested: " << widget.suggested_action << "\n";
std::cout << " State: " << (widget.visible ? "visible" : "hidden")
<< ", " << (widget.enabled ? "enabled" : "disabled") << "\n";
std::cout << absl::StrFormat(" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n",
widget.bounds.min_x, widget.bounds.min_y,
widget.bounds.max_x, widget.bounds.max_y);
if (widget.has_bounds) {
std::cout << absl::StrFormat(
" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n", widget.bounds.min_x,
widget.bounds.min_y, widget.bounds.max_x, widget.bounds.max_y);
} else {
std::cout << " Bounds: (not available)\n";
}
std::cout << " Last seen: frame " << widget.last_seen_frame;
if (widget.last_seen_at.has_value()) {
std::cout << " @ "
<< absl::FormatTime("%Y-%m-%d %H:%M:%S",
*widget.last_seen_at,
absl::LocalTimeZone());
}
if (widget.stale) {
std::cout << " [STALE]";
}
std::cout << "\n";
std::cout << " Widget ID: 0x" << std::hex << widget.widget_id
<< std::dec << "\n";
}

16
src/cli/service/gui_automation_client.cc
View File

@@ -519,11 +519,19 @@ absl::StatusOr<DiscoverWidgetsResult> GuiAutomationClient::DiscoverWidgets(
widget.suggested_action = widget_proto.suggested_action();
widget.visible = widget_proto.visible();
widget.enabled = widget_proto.enabled();
widget.bounds.min_x = widget_proto.bounds().min_x();
widget.bounds.min_y = widget_proto.bounds().min_y();
widget.bounds.max_x = widget_proto.bounds().max_x();
widget.bounds.max_y = widget_proto.bounds().max_y();
widget.has_bounds = widget_proto.has_bounds();
if (widget.has_bounds) {
widget.bounds.min_x = widget_proto.bounds().min_x();
widget.bounds.min_y = widget_proto.bounds().min_y();
widget.bounds.max_x = widget_proto.bounds().max_x();
widget.bounds.max_y = widget_proto.bounds().max_y();
} else {
widget.bounds = WidgetBoundingBox();
}
widget.widget_id = widget_proto.widget_id();
widget.last_seen_frame = widget_proto.last_seen_frame();
widget.last_seen_at = OptionalTimeFromMillis(widget_proto.last_seen_at_ms());
widget.stale = widget_proto.stale();
window_info.widgets.push_back(std::move(widget));
}

4
src/cli/service/gui_automation_client.h
View File

@@ -152,6 +152,10 @@ struct WidgetDescriptor {
bool enabled = true;
WidgetBoundingBox bounds;
uint32_t widget_id = 0;
bool has_bounds = false;
int64_t last_seen_frame = -1;
std::optional<absl::Time> last_seen_at;
bool stale = false;
};
struct DiscoveredWindowInfo {