feat: Enhance widget discovery with telemetry data and improve output formatting
This commit is contained in:
@@ -202,31 +202,48 @@ z3ed gui discover --window "Overworld"
|
|||||||
z3ed gui discover --type button
|
z3ed gui discover --type button
|
||||||
```
|
```
|
||||||
|
|
||||||
**API Schema**:
|
**API Schema (current)**:
|
||||||
```proto
|
```proto
|
||||||
message DiscoverWidgetsRequest {
|
message DiscoverWidgetsRequest {
|
||||||
string window_filter = 1; // Optional: filter by window name
|
string window_filter = 1;
|
||||||
enum WidgetType { ALL = 0; BUTTON = 1; INPUT = 2; MENU = 3; TAB = 4; CHECKBOX = 5; }
|
|
||||||
WidgetType type_filter = 2;
|
WidgetType type_filter = 2;
|
||||||
|
string path_prefix = 3;
|
||||||
|
bool include_invisible = 4;
|
||||||
|
bool include_disabled = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
message WidgetBounds {
|
||||||
|
float min_x = 1;
|
||||||
|
float min_y = 2;
|
||||||
|
float max_x = 3;
|
||||||
|
float max_y = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DiscoveredWidget {
|
||||||
|
string path = 1;
|
||||||
|
string label = 2;
|
||||||
|
string type = 3;
|
||||||
|
string description = 4;
|
||||||
|
string suggested_action = 5;
|
||||||
|
bool visible = 6;
|
||||||
|
bool enabled = 7;
|
||||||
|
WidgetBounds bounds = 8;
|
||||||
|
uint32 widget_id = 9;
|
||||||
|
int64 last_seen_frame = 10;
|
||||||
|
int64 last_seen_at_ms = 11;
|
||||||
|
bool stale = 12;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DiscoveredWindow {
|
||||||
|
string name = 1;
|
||||||
|
bool visible = 2;
|
||||||
|
repeated DiscoveredWidget widgets = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
message DiscoverWidgetsResponse {
|
message DiscoverWidgetsResponse {
|
||||||
repeated WindowInfo windows = 1;
|
repeated DiscoveredWindow windows = 1;
|
||||||
}
|
int32 total_widgets = 2;
|
||||||
|
int64 generated_at_ms = 3;
|
||||||
message WindowInfo {
|
|
||||||
string name = 1;
|
|
||||||
bool is_visible = 2;
|
|
||||||
repeated WidgetInfo widgets = 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
message WidgetInfo {
|
|
||||||
string id = 1;
|
|
||||||
string label = 2;
|
|
||||||
string type = 3; // "button", "input", "menu", etc.
|
|
||||||
bool is_enabled = 4;
|
|
||||||
string position = 5; // "x,y,width,height"
|
|
||||||
string suggested_action = 6; // "Click button:Open ROM"
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -266,6 +266,12 @@ Options:
|
|||||||
--format <mode> Output as `table` (default) or `json`
|
--format <mode> Output as `table` (default) or `json`
|
||||||
--limit <n> Maximum widgets to display (useful for large UIs)
|
--limit <n> Maximum widgets to display (useful for large UIs)
|
||||||
|
|
||||||
|
Each discovered widget now reports:
|
||||||
|
- Current visibility/enablement and bounding box (when available)
|
||||||
|
- Last observed frame number and UTC timestamp
|
||||||
|
- A `stale` flag when the widget hasn't appeared in the current frame
|
||||||
|
- Its underlying ImGui ID for low-level automation
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
# Discover all widgets currently registered
|
# Discover all widgets currently registered
|
||||||
z3ed agent gui discover
|
z3ed agent gui discover
|
||||||
@@ -292,10 +298,13 @@ Window: Overworld (visible)
|
|||||||
Suggested: Click button:Save
|
Suggested: Click button:Save
|
||||||
State: visible, enabled
|
State: visible, enabled
|
||||||
Bounds: (24.0, 64.0) → (112.0, 92.0)
|
Bounds: (24.0, 64.0) → (112.0, 92.0)
|
||||||
|
Last seen: frame 18432 @ 2025-01-16 19:42:05
|
||||||
Widget ID: 0x13fc41a2
|
Widget ID: 0x13fc41a2
|
||||||
|
|
||||||
Widgets shown: 3 of 18 (truncated)
|
Widgets shown: 3 of 18 (truncated)
|
||||||
Snapshot: 2025-01-16 19:42:05
|
Snapshot: 2025-01-16 19:42:05
|
||||||
|
|
||||||
|
_Widgets that have not appeared in the current frame are marked `[STALE]` in the table output._
|
||||||
```
|
```
|
||||||
|
|
||||||
**Use Cases**:
|
**Use Cases**:
|
||||||
|
|||||||
@@ -121,7 +121,7 @@ z3ed agent test results --test-id grpc_click_12345678 --include-logs
|
|||||||
z3ed agent test list --category grpc
|
z3ed agent test list --category grpc
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Widget Discovery (IT-06) 🔜 PLANNED
|
#### Widget Discovery (IT-06) <EFBFBD> IN PROGRESS — telemetry available
|
||||||
```bash
|
```bash
|
||||||
# Discover all widgets
|
# Discover all widgets
|
||||||
z3ed agent gui discover
|
z3ed agent gui discover
|
||||||
@@ -129,8 +129,8 @@ z3ed agent gui discover
|
|||||||
# Filter by window
|
# Filter by window
|
||||||
z3ed agent gui discover --window "Overworld"
|
z3ed agent gui discover --window "Overworld"
|
||||||
|
|
||||||
# Get only buttons
|
# Get only buttons and include hidden/disabled widgets for AI diffing
|
||||||
z3ed agent gui discover --type button --format json
|
z3ed agent gui discover --type button --include-invisible --include-disabled --format json
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Test Recording (IT-07) 🔜 PLANNED
|
#### Test Recording (IT-07) 🔜 PLANNED
|
||||||
@@ -365,7 +365,7 @@ grpcurl ... Wait '{"condition":"window_visible:WindowName"}'
|
|||||||
# 3. Assert widget exists
|
# 3. Assert widget exists
|
||||||
grpcurl ... Assert '{"condition":"exists:button:XYZ"}'
|
grpcurl ... Assert '{"condition":"exists:button:XYZ"}'
|
||||||
|
|
||||||
# 4. Use widget discovery (IT-06, planned)
|
# 4. Use widget discovery (IT-06 telemetry)
|
||||||
z3ed agent gui discover --window "WindowName"
|
z3ed agent gui discover --window "WindowName"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
#include "app/editor/editor_manager.h"
|
#include "app/editor/editor_manager.h"
|
||||||
#include "app/gui/background_renderer.h"
|
#include "app/gui/background_renderer.h"
|
||||||
#include "app/gui/theme_manager.h"
|
#include "app/gui/theme_manager.h"
|
||||||
|
#include "app/gui/widget_id_registry.h"
|
||||||
#include "imgui/backends/imgui_impl_sdl2.h"
|
#include "imgui/backends/imgui_impl_sdl2.h"
|
||||||
#include "imgui/backends/imgui_impl_sdlrenderer2.h"
|
#include "imgui/backends/imgui_impl_sdlrenderer2.h"
|
||||||
#include "imgui/imgui.h"
|
#include "imgui/imgui.h"
|
||||||
@@ -65,7 +66,10 @@ absl::Status Controller::OnLoad() {
|
|||||||
|
|
||||||
ImGui::End();
|
ImGui::End();
|
||||||
#endif
|
#endif
|
||||||
RETURN_IF_ERROR(editor_manager_.Update());
|
gui::WidgetIdRegistry::Instance().BeginFrame();
|
||||||
|
absl::Status update_status = editor_manager_.Update();
|
||||||
|
gui::WidgetIdRegistry::Instance().EndFrame();
|
||||||
|
RETURN_IF_ERROR(update_status);
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -290,6 +290,9 @@ message DiscoveredWidget {
|
|||||||
bool enabled = 7; // Currently enabled for interaction
|
bool enabled = 7; // Currently enabled for interaction
|
||||||
WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates
|
WidgetBounds bounds = 8; // Bounding rectangle in screen coordinates
|
||||||
uint32 widget_id = 9; // ImGui ID (debugging / direct access)
|
uint32 widget_id = 9; // ImGui ID (debugging / direct access)
|
||||||
|
int64 last_seen_frame = 10; // Frame number when widget was last observed
|
||||||
|
int64 last_seen_at_ms = 11; // Wall-clock timestamp of last observation
|
||||||
|
bool stale = 12; // True if widget not seen in the current frame
|
||||||
}
|
}
|
||||||
|
|
||||||
message DiscoveredWindow {
|
message DiscoveredWindow {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
#include "app/core/service/widget_discovery_service.h"
|
#include "app/core/service/widget_discovery_service.h"
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <cstdint>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
@@ -17,7 +18,7 @@ namespace {
|
|||||||
|
|
||||||
struct WindowEntry {
|
struct WindowEntry {
|
||||||
int index = -1;
|
int index = -1;
|
||||||
bool visible = true;
|
bool visible = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
@@ -57,39 +58,17 @@ void WidgetDiscoveryService::CollectWidgets(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::string window_name = ExtractWindowName(path);
|
const std::string window_name =
|
||||||
|
info.window_name.empty() ? ExtractWindowName(path) : info.window_name;
|
||||||
if (!MatchesWindow(window_name, window_filter_lower)) {
|
if (!MatchesWindow(window_name, window_filter_lower)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
|
const std::string label =
|
||||||
WindowEntry& entry = it->second;
|
info.label.empty() ? ExtractLabel(path) : info.label;
|
||||||
|
|
||||||
if (inserted) {
|
bool widget_enabled = info.enabled;
|
||||||
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
bool widget_visible = info.visible;
|
||||||
if (ctx) {
|
|
||||||
ImGuiTestItemInfo window_info =
|
|
||||||
ctx->WindowInfo(window_name.c_str(), ImGuiTestOpFlags_NoError);
|
|
||||||
entry.visible = (window_info.ID != 0);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!include_invisible && !entry.visible) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (entry.index == -1) {
|
|
||||||
DiscoveredWindow* window_proto = response->add_windows();
|
|
||||||
entry.index = response->windows_size() - 1;
|
|
||||||
window_proto->set_name(window_name);
|
|
||||||
window_proto->set_visible(entry.visible);
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string label = ExtractLabel(path);
|
|
||||||
|
|
||||||
bool widget_enabled = true;
|
|
||||||
bool widget_visible = entry.visible;
|
|
||||||
|
|
||||||
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||||
bool has_item_info = false;
|
bool has_item_info = false;
|
||||||
@@ -103,8 +82,18 @@ void WidgetDiscoveryService::CollectWidgets(
|
|||||||
widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0;
|
widget_enabled = (item_info.ItemFlags & ImGuiItemFlags_Disabled) == 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#else
|
||||||
|
(void)ctx;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
auto [it, inserted] = window_lookup.emplace(window_name, WindowEntry{});
|
||||||
|
WindowEntry& entry = it->second;
|
||||||
|
if (inserted) {
|
||||||
|
entry.visible = widget_visible;
|
||||||
|
} else {
|
||||||
|
entry.visible = entry.visible || widget_visible;
|
||||||
|
}
|
||||||
|
|
||||||
if (!include_invisible && !widget_visible) {
|
if (!include_invisible && !widget_visible) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -112,7 +101,16 @@ void WidgetDiscoveryService::CollectWidgets(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (entry.index == -1) {
|
||||||
|
DiscoveredWindow* window_proto = response->add_windows();
|
||||||
|
entry.index = response->windows_size() - 1;
|
||||||
|
window_proto->set_name(window_name);
|
||||||
|
window_proto->set_visible(entry.visible);
|
||||||
|
}
|
||||||
|
|
||||||
auto* window_proto = response->mutable_windows(entry.index);
|
auto* window_proto = response->mutable_windows(entry.index);
|
||||||
|
window_proto->set_visible(entry.visible);
|
||||||
|
|
||||||
auto* widget_proto = window_proto->add_widgets();
|
auto* widget_proto = window_proto->add_widgets();
|
||||||
widget_proto->set_path(path);
|
widget_proto->set_path(path);
|
||||||
widget_proto->set_label(label);
|
widget_proto->set_label(label);
|
||||||
@@ -126,19 +124,34 @@ void WidgetDiscoveryService::CollectWidgets(
|
|||||||
widget_proto->set_description(info.description);
|
widget_proto->set_description(info.description);
|
||||||
}
|
}
|
||||||
|
|
||||||
WidgetBounds* bounds = widget_proto->mutable_bounds();
|
if (info.bounds.valid) {
|
||||||
|
WidgetBounds* bounds = widget_proto->mutable_bounds();
|
||||||
|
bounds->set_min_x(info.bounds.min_x);
|
||||||
|
bounds->set_min_y(info.bounds.min_y);
|
||||||
|
bounds->set_max_x(info.bounds.max_x);
|
||||||
|
bounds->set_max_y(info.bounds.max_y);
|
||||||
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
|
||||||
if (ctx && has_item_info) {
|
} else if (ctx && has_item_info) {
|
||||||
|
WidgetBounds* bounds = widget_proto->mutable_bounds();
|
||||||
bounds->set_min_x(item_info.RectFull.Min.x);
|
bounds->set_min_x(item_info.RectFull.Min.x);
|
||||||
bounds->set_min_y(item_info.RectFull.Min.y);
|
bounds->set_min_y(item_info.RectFull.Min.y);
|
||||||
bounds->set_max_x(item_info.RectFull.Max.x);
|
bounds->set_max_x(item_info.RectFull.Max.x);
|
||||||
bounds->set_max_y(item_info.RectFull.Max.y);
|
bounds->set_max_y(item_info.RectFull.Max.y);
|
||||||
} else {
|
} else {
|
||||||
(void)ctx;
|
(void)ctx;
|
||||||
}
|
|
||||||
#else
|
#else
|
||||||
(void)ctx;
|
} else {
|
||||||
|
(void)ctx;
|
||||||
#endif
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
widget_proto->set_last_seen_frame(info.last_seen_frame);
|
||||||
|
int64_t last_seen_ms = 0;
|
||||||
|
if (info.last_seen_time != absl::Time()) {
|
||||||
|
last_seen_ms = absl::ToUnixMillis(info.last_seen_time);
|
||||||
|
}
|
||||||
|
widget_proto->set_last_seen_at_ms(last_seen_ms);
|
||||||
|
widget_proto->set_stale(info.stale_frame_count > 0);
|
||||||
|
|
||||||
++total_widgets;
|
++total_widgets;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,13 +1,19 @@
|
|||||||
#include "widget_id_registry.h"
|
#include "widget_id_registry.h"
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <chrono>
|
||||||
|
#include <cstdio>
|
||||||
|
#include <ctime>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
|
||||||
|
#include "absl/strings/ascii.h"
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "absl/strings/str_format.h"
|
#include "absl/strings/str_format.h"
|
||||||
#include "absl/strings/str_join.h"
|
#include "absl/strings/str_join.h"
|
||||||
#include "absl/strings/str_split.h"
|
#include "absl/strings/str_split.h"
|
||||||
|
#include "absl/strings/strip.h"
|
||||||
|
#include "absl/time/clock.h"
|
||||||
#include "imgui/imgui_internal.h" // For ImGuiContext internals
|
#include "imgui/imgui_internal.h" // For ImGuiContext internals
|
||||||
|
|
||||||
namespace yaze {
|
namespace yaze {
|
||||||
@@ -54,11 +60,170 @@ WidgetIdRegistry& WidgetIdRegistry::Instance() {
|
|||||||
return instance;
|
return instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void WidgetIdRegistry::BeginFrame() {
|
||||||
|
ImGuiContext* ctx = ImGui::GetCurrentContext();
|
||||||
|
if (ctx) {
|
||||||
|
current_frame_ = ctx->FrameCount;
|
||||||
|
} else if (current_frame_ >= 0) {
|
||||||
|
++current_frame_;
|
||||||
|
} else {
|
||||||
|
current_frame_ = 0;
|
||||||
|
}
|
||||||
|
frame_time_ = absl::Now();
|
||||||
|
for (auto& [_, info] : widgets_) {
|
||||||
|
info.seen_in_current_frame = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void WidgetIdRegistry::EndFrame() {
|
||||||
|
for (auto& [_, info] : widgets_) {
|
||||||
|
if (!info.seen_in_current_frame) {
|
||||||
|
info.visible = false;
|
||||||
|
info.enabled = false;
|
||||||
|
info.bounds.valid = false;
|
||||||
|
info.stale_frame_count += 1;
|
||||||
|
} else {
|
||||||
|
info.seen_in_current_frame = false;
|
||||||
|
info.stale_frame_count = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TrimStaleEntries();
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
std::string ExtractWindowFromPath(absl::string_view path) {
|
||||||
|
size_t slash = path.find('/');
|
||||||
|
if (slash == absl::string_view::npos) {
|
||||||
|
return std::string(path);
|
||||||
|
}
|
||||||
|
return std::string(path.substr(0, slash));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string ExtractLabelFromPath(absl::string_view path) {
|
||||||
|
size_t colon = path.rfind(':');
|
||||||
|
if (colon == absl::string_view::npos) {
|
||||||
|
size_t slash = path.rfind('/');
|
||||||
|
if (slash == absl::string_view::npos) {
|
||||||
|
return std::string(path);
|
||||||
|
}
|
||||||
|
return std::string(path.substr(slash + 1));
|
||||||
|
}
|
||||||
|
return std::string(path.substr(colon + 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string FormatTimestampUTC(const absl::Time& timestamp) {
|
||||||
|
if (timestamp == absl::Time()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::chrono::system_clock::time_point chrono_time =
|
||||||
|
absl::ToChronoTime(timestamp);
|
||||||
|
std::time_t time_value = std::chrono::system_clock::to_time_t(chrono_time);
|
||||||
|
|
||||||
|
std::tm tm_buffer;
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (gmtime_s(&tm_buffer, &time_value) != 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
if (gmtime_r(&time_value, &tm_buffer) == nullptr) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
char buffer[32];
|
||||||
|
if (std::snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02dZ",
|
||||||
|
tm_buffer.tm_year + 1900, tm_buffer.tm_mon + 1,
|
||||||
|
tm_buffer.tm_mday, tm_buffer.tm_hour, tm_buffer.tm_min,
|
||||||
|
tm_buffer.tm_sec) <= 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
return std::string(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
WidgetIdRegistry::WidgetBounds BoundsFromImGui(const ImRect& rect) {
|
||||||
|
WidgetIdRegistry::WidgetBounds bounds;
|
||||||
|
bounds.min_x = rect.Min.x;
|
||||||
|
bounds.min_y = rect.Min.y;
|
||||||
|
bounds.max_x = rect.Max.x;
|
||||||
|
bounds.max_y = rect.Max.y;
|
||||||
|
bounds.valid = true;
|
||||||
|
return bounds;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
void WidgetIdRegistry::RegisterWidget(const std::string& full_path,
|
void WidgetIdRegistry::RegisterWidget(const std::string& full_path,
|
||||||
const std::string& type, ImGuiID imgui_id,
|
const std::string& type,
|
||||||
const std::string& description) {
|
ImGuiID imgui_id,
|
||||||
WidgetInfo info{full_path, type, imgui_id, description};
|
const std::string& description,
|
||||||
widgets_[full_path] = info;
|
const WidgetMetadata& metadata) {
|
||||||
|
WidgetInfo& info = widgets_[full_path];
|
||||||
|
info.full_path = full_path;
|
||||||
|
info.type = type;
|
||||||
|
info.imgui_id = imgui_id;
|
||||||
|
info.description = description;
|
||||||
|
|
||||||
|
if (metadata.label.has_value()) {
|
||||||
|
info.label = NormalizeLabel(*metadata.label);
|
||||||
|
} else {
|
||||||
|
info.label = NormalizeLabel(ExtractLabelFromPath(full_path));
|
||||||
|
}
|
||||||
|
if (info.label.empty()) {
|
||||||
|
info.label = ExtractLabelFromPath(full_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metadata.window_name.has_value()) {
|
||||||
|
info.window_name = NormalizeLabel(*metadata.window_name);
|
||||||
|
} else {
|
||||||
|
info.window_name = NormalizePathSegment(ExtractWindowFromPath(full_path));
|
||||||
|
}
|
||||||
|
if (info.window_name.empty()) {
|
||||||
|
info.window_name = ExtractWindowFromPath(full_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
ImGuiContext* ctx = ImGui::GetCurrentContext();
|
||||||
|
absl::Time observed_at = absl::Now();
|
||||||
|
|
||||||
|
if (ctx) {
|
||||||
|
const ImGuiLastItemData& last = ctx->LastItemData;
|
||||||
|
if (metadata.visible.has_value()) {
|
||||||
|
info.visible = *metadata.visible;
|
||||||
|
} else {
|
||||||
|
info.visible = (last.StatusFlags & ImGuiItemStatusFlags_Visible) != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metadata.enabled.has_value()) {
|
||||||
|
info.enabled = *metadata.enabled;
|
||||||
|
} else {
|
||||||
|
info.enabled = (last.ItemFlags & ImGuiItemFlags_Disabled) == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metadata.bounds.has_value()) {
|
||||||
|
info.bounds = *metadata.bounds;
|
||||||
|
} else {
|
||||||
|
info.bounds = BoundsFromImGui(last.Rect);
|
||||||
|
}
|
||||||
|
|
||||||
|
info.last_seen_frame = ctx->FrameCount;
|
||||||
|
} else {
|
||||||
|
info.visible = metadata.visible.value_or(true);
|
||||||
|
info.enabled = metadata.enabled.value_or(true);
|
||||||
|
if (metadata.bounds.has_value()) {
|
||||||
|
info.bounds = *metadata.bounds;
|
||||||
|
} else {
|
||||||
|
info.bounds.valid = false;
|
||||||
|
}
|
||||||
|
if (current_frame_ >= 0) {
|
||||||
|
info.last_seen_frame = current_frame_;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
info.last_seen_time = observed_at;
|
||||||
|
info.seen_in_current_frame = true;
|
||||||
|
info.stale_frame_count = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<std::string> WidgetIdRegistry::FindWidgets(
|
std::vector<std::string> WidgetIdRegistry::FindWidgets(
|
||||||
@@ -113,7 +278,10 @@ const WidgetIdRegistry::WidgetInfo* WidgetIdRegistry::GetWidgetInfo(
|
|||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
void WidgetIdRegistry::Clear() { widgets_.clear(); }
|
void WidgetIdRegistry::Clear() {
|
||||||
|
widgets_.clear();
|
||||||
|
current_frame_ = -1;
|
||||||
|
}
|
||||||
|
|
||||||
std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const {
|
std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const {
|
||||||
std::ostringstream ss;
|
std::ostringstream ss;
|
||||||
@@ -130,12 +298,36 @@ std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const {
|
|||||||
ss << " {\n";
|
ss << " {\n";
|
||||||
ss << absl::StrFormat(" \"path\": \"%s\",\n", path);
|
ss << absl::StrFormat(" \"path\": \"%s\",\n", path);
|
||||||
ss << absl::StrFormat(" \"type\": \"%s\",\n", info.type);
|
ss << absl::StrFormat(" \"type\": \"%s\",\n", info.type);
|
||||||
ss << absl::StrFormat(" \"imgui_id\": %u", info.imgui_id);
|
ss << absl::StrFormat(" \"imgui_id\": %u,\n", info.imgui_id);
|
||||||
|
ss << absl::StrFormat(" \"label\": \"%s\",\n", info.label);
|
||||||
|
ss << absl::StrFormat(" \"window\": \"%s\",\n", info.window_name);
|
||||||
|
ss << absl::StrFormat(" \"visible\": %s,\n",
|
||||||
|
info.visible ? "true" : "false");
|
||||||
|
ss << absl::StrFormat(" \"enabled\": %s,\n",
|
||||||
|
info.enabled ? "true" : "false");
|
||||||
|
if (info.bounds.valid) {
|
||||||
|
ss << absl::StrFormat(
|
||||||
|
" \"bounds\": {\"min\": [%0.1f, %0.1f], \"max\": [%0.1f, %0.1f]},\n",
|
||||||
|
info.bounds.min_x, info.bounds.min_y, info.bounds.max_x,
|
||||||
|
info.bounds.max_y);
|
||||||
|
} else {
|
||||||
|
ss << " \"bounds\": null,\n";
|
||||||
|
}
|
||||||
|
ss << absl::StrFormat(" \"last_seen_frame\": %d,\n",
|
||||||
|
info.last_seen_frame);
|
||||||
|
std::string iso_timestamp = FormatTimestampUTC(info.last_seen_time);
|
||||||
|
ss << absl::StrFormat(" \"last_seen_at\": \"%s\",\n",
|
||||||
|
iso_timestamp);
|
||||||
|
ss << absl::StrFormat(" \"stale\": %s",
|
||||||
|
info.stale_frame_count > 0 ? "true" : "false");
|
||||||
if (!info.description.empty()) {
|
if (!info.description.empty()) {
|
||||||
ss << ",\n";
|
ss << ",\n";
|
||||||
ss << absl::StrFormat(" \"description\": \"%s\"", info.description);
|
ss << absl::StrFormat(" \"description\": \"%s\"\n",
|
||||||
|
info.description);
|
||||||
|
} else {
|
||||||
|
ss << "\n";
|
||||||
}
|
}
|
||||||
ss << "\n }";
|
ss << " }";
|
||||||
}
|
}
|
||||||
|
|
||||||
ss << "\n ]\n";
|
ss << "\n ]\n";
|
||||||
@@ -148,6 +340,22 @@ std::string WidgetIdRegistry::ExportCatalog(const std::string& format) const {
|
|||||||
ss << absl::StrFormat(" - path: \"%s\"\n", path);
|
ss << absl::StrFormat(" - path: \"%s\"\n", path);
|
||||||
ss << absl::StrFormat(" type: %s\n", info.type);
|
ss << absl::StrFormat(" type: %s\n", info.type);
|
||||||
ss << absl::StrFormat(" imgui_id: %u\n", info.imgui_id);
|
ss << absl::StrFormat(" imgui_id: %u\n", info.imgui_id);
|
||||||
|
ss << absl::StrFormat(" label: \"%s\"\n", info.label);
|
||||||
|
ss << absl::StrFormat(" window: \"%s\"\n", info.window_name);
|
||||||
|
ss << absl::StrFormat(" visible: %s\n", info.visible ? "true" : "false");
|
||||||
|
ss << absl::StrFormat(" enabled: %s\n", info.enabled ? "true" : "false");
|
||||||
|
if (info.bounds.valid) {
|
||||||
|
ss << " bounds:\n";
|
||||||
|
ss << absl::StrFormat(" min: [%0.1f, %0.1f]\n", info.bounds.min_x,
|
||||||
|
info.bounds.min_y);
|
||||||
|
ss << absl::StrFormat(" max: [%0.1f, %0.1f]\n", info.bounds.max_x,
|
||||||
|
info.bounds.max_y);
|
||||||
|
}
|
||||||
|
ss << absl::StrFormat(" last_seen_frame: %d\n", info.last_seen_frame);
|
||||||
|
std::string iso_timestamp = FormatTimestampUTC(info.last_seen_time);
|
||||||
|
ss << absl::StrFormat(" last_seen_at: %s\n", iso_timestamp);
|
||||||
|
ss << absl::StrFormat(" stale: %s\n",
|
||||||
|
info.stale_frame_count > 0 ? "true" : "false");
|
||||||
|
|
||||||
// Parse hierarchical context from path
|
// Parse hierarchical context from path
|
||||||
std::vector<std::string> segments = absl::StrSplit(path, '/');
|
std::vector<std::string> segments = absl::StrSplit(path, '/');
|
||||||
@@ -200,5 +408,39 @@ void WidgetIdRegistry::ExportCatalogToFile(const std::string& output_file,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string WidgetIdRegistry::NormalizeLabel(absl::string_view label) {
|
||||||
|
size_t pos = label.find("##");
|
||||||
|
if (pos != absl::string_view::npos) {
|
||||||
|
label = label.substr(0, pos);
|
||||||
|
}
|
||||||
|
std::string sanitized = std::string(absl::StripAsciiWhitespace(label));
|
||||||
|
return sanitized;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string WidgetIdRegistry::NormalizePathSegment(absl::string_view segment) {
|
||||||
|
return NormalizeLabel(segment);
|
||||||
|
}
|
||||||
|
|
||||||
|
void WidgetIdRegistry::TrimStaleEntries() {
|
||||||
|
auto it = widgets_.begin();
|
||||||
|
while (it != widgets_.end()) {
|
||||||
|
if (ShouldPrune(it->second)) {
|
||||||
|
it = widgets_.erase(it);
|
||||||
|
} else {
|
||||||
|
++it;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool WidgetIdRegistry::ShouldPrune(const WidgetInfo& info) const {
|
||||||
|
if (info.last_seen_frame < 0 || stale_frame_limit_ <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (current_frame_ < 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (current_frame_ - info.last_seen_frame) > stale_frame_limit_;
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace gui
|
} // namespace gui
|
||||||
} // namespace yaze
|
} // namespace yaze
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
#ifndef YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_
|
#ifndef YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_
|
||||||
#define YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_
|
#define YAZE_APP_GUI_WIDGET_ID_REGISTRY_H_
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
|
#include <optional>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include "absl/strings/string_view.h"
|
||||||
|
#include "absl/time/time.h"
|
||||||
#include "imgui/imgui.h"
|
#include "imgui/imgui.h"
|
||||||
|
|
||||||
namespace yaze {
|
namespace yaze {
|
||||||
@@ -62,20 +66,50 @@ class WidgetIdScope {
|
|||||||
*/
|
*/
|
||||||
class WidgetIdRegistry {
|
class WidgetIdRegistry {
|
||||||
public:
|
public:
|
||||||
|
struct WidgetBounds {
|
||||||
|
float min_x = 0.0f;
|
||||||
|
float min_y = 0.0f;
|
||||||
|
float max_x = 0.0f;
|
||||||
|
float max_y = 0.0f;
|
||||||
|
bool valid = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct WidgetMetadata {
|
||||||
|
std::optional<std::string> label;
|
||||||
|
std::optional<std::string> window_name;
|
||||||
|
std::optional<bool> visible;
|
||||||
|
std::optional<bool> enabled;
|
||||||
|
std::optional<WidgetBounds> bounds;
|
||||||
|
};
|
||||||
|
|
||||||
struct WidgetInfo {
|
struct WidgetInfo {
|
||||||
std::string full_path; // e.g. "Overworld/Canvas/canvas:Map"
|
std::string full_path; // e.g. "Overworld/Canvas/canvas:Map"
|
||||||
std::string type; // e.g. "button", "input", "canvas", "table"
|
std::string type; // e.g. "button", "input", "canvas", "table"
|
||||||
ImGuiID imgui_id; // ImGui's internal ID
|
ImGuiID imgui_id; // ImGui's internal ID
|
||||||
std::string description; // Optional human-readable description
|
std::string description; // Optional human-readable description
|
||||||
|
std::string label; // Sanitized display label (without IDs/icons)
|
||||||
|
std::string window_name; // Window this widget was last seen in
|
||||||
|
bool visible = true; // Visibility in the most recent frame
|
||||||
|
bool enabled = true; // Enabled state in the most recent frame
|
||||||
|
WidgetBounds bounds; // Bounding box in screen space
|
||||||
|
int last_seen_frame = -1;
|
||||||
|
absl::Time last_seen_time;
|
||||||
|
bool seen_in_current_frame = false;
|
||||||
|
int stale_frame_count = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
static WidgetIdRegistry& Instance();
|
static WidgetIdRegistry& Instance();
|
||||||
|
|
||||||
|
// Frame lifecycle - call once per ImGui frame
|
||||||
|
void BeginFrame();
|
||||||
|
void EndFrame();
|
||||||
|
|
||||||
// Register a widget for discovery
|
// Register a widget for discovery
|
||||||
// Should be called after widget is created (when ImGui::GetItemID() is valid)
|
// Should be called after widget is created (when ImGui::GetItemID() is valid)
|
||||||
void RegisterWidget(const std::string& full_path, const std::string& type,
|
void RegisterWidget(const std::string& full_path, const std::string& type,
|
||||||
ImGuiID imgui_id,
|
ImGuiID imgui_id,
|
||||||
const std::string& description = "");
|
const std::string& description = "",
|
||||||
|
const WidgetMetadata& metadata = WidgetMetadata());
|
||||||
|
|
||||||
// Query widgets for test automation
|
// Query widgets for test automation
|
||||||
std::vector<std::string> FindWidgets(const std::string& pattern) const;
|
std::vector<std::string> FindWidgets(const std::string& pattern) const;
|
||||||
@@ -96,9 +130,19 @@ class WidgetIdRegistry {
|
|||||||
void ExportCatalogToFile(const std::string& output_file,
|
void ExportCatalogToFile(const std::string& output_file,
|
||||||
const std::string& format = "yaml") const;
|
const std::string& format = "yaml") const;
|
||||||
|
|
||||||
|
// Helper utilities for consistent naming
|
||||||
|
static std::string NormalizeLabel(absl::string_view label);
|
||||||
|
static std::string NormalizePathSegment(absl::string_view segment);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
WidgetIdRegistry() = default;
|
WidgetIdRegistry() = default;
|
||||||
|
void TrimStaleEntries();
|
||||||
|
bool ShouldPrune(const WidgetInfo& info) const;
|
||||||
|
|
||||||
std::unordered_map<std::string, WidgetInfo> widgets_;
|
std::unordered_map<std::string, WidgetInfo> widgets_;
|
||||||
|
int current_frame_ = -1;
|
||||||
|
absl::Time frame_time_;
|
||||||
|
int stale_frame_limit_ = 600; // frames before pruning a widget
|
||||||
};
|
};
|
||||||
|
|
||||||
// RAII helper macros for convenient scoping
|
// RAII helper macros for convenient scoping
|
||||||
|
|||||||
@@ -227,11 +227,31 @@ absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
|
|||||||
<< (widget.visible ? "true" : "false") << ",\n";
|
<< (widget.visible ? "true" : "false") << ",\n";
|
||||||
std::cout << " \"enabled\": "
|
std::cout << " \"enabled\": "
|
||||||
<< (widget.enabled ? "true" : "false") << ",\n";
|
<< (widget.enabled ? "true" : "false") << ",\n";
|
||||||
std::cout << " \"bounds\": { \"min\": [" << widget.bounds.min_x
|
if (widget.has_bounds) {
|
||||||
<< ", " << widget.bounds.min_y << "], \"max\": ["
|
std::cout << " \"bounds\": { \"min\": ["
|
||||||
<< widget.bounds.max_x << ", " << widget.bounds.max_y
|
<< widget.bounds.min_x << ", " << widget.bounds.min_y
|
||||||
<< "] },\n";
|
<< "], \"max\": [" << widget.bounds.max_x << ", "
|
||||||
std::cout << " \"widgetId\": " << widget.widget_id << "\n";
|
<< widget.bounds.max_y << "] },\n";
|
||||||
|
} else {
|
||||||
|
std::cout << " \"bounds\": null,\n";
|
||||||
|
}
|
||||||
|
std::cout << " \"widgetId\": " << widget.widget_id << ",\n";
|
||||||
|
std::cout << " \"lastSeenFrame\": "
|
||||||
|
<< widget.last_seen_frame << ",\n";
|
||||||
|
std::cout << " \"lastSeenAt\": ";
|
||||||
|
if (widget.last_seen_at.has_value()) {
|
||||||
|
std::cout
|
||||||
|
<< "\""
|
||||||
|
<< JsonEscape(absl::FormatTime("%Y-%m-%dT%H:%M:%SZ",
|
||||||
|
*widget.last_seen_at,
|
||||||
|
absl::UTCTimeZone()))
|
||||||
|
<< "\"";
|
||||||
|
} else {
|
||||||
|
std::cout << "null";
|
||||||
|
}
|
||||||
|
std::cout << ",\n";
|
||||||
|
std::cout << " \"stale\": "
|
||||||
|
<< (widget.stale ? "true" : "false") << "\n";
|
||||||
std::cout << " }";
|
std::cout << " }";
|
||||||
if (i + 1 < window.widgets.size()) {
|
if (i + 1 < window.widgets.size()) {
|
||||||
std::cout << ",";
|
std::cout << ",";
|
||||||
@@ -284,9 +304,24 @@ absl::Status HandleGuiDiscoverCommand(const std::vector<std::string>& arg_vec) {
|
|||||||
std::cout << " Suggested: " << widget.suggested_action << "\n";
|
std::cout << " Suggested: " << widget.suggested_action << "\n";
|
||||||
std::cout << " State: " << (widget.visible ? "visible" : "hidden")
|
std::cout << " State: " << (widget.visible ? "visible" : "hidden")
|
||||||
<< ", " << (widget.enabled ? "enabled" : "disabled") << "\n";
|
<< ", " << (widget.enabled ? "enabled" : "disabled") << "\n";
|
||||||
std::cout << absl::StrFormat(" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n",
|
if (widget.has_bounds) {
|
||||||
widget.bounds.min_x, widget.bounds.min_y,
|
std::cout << absl::StrFormat(
|
||||||
widget.bounds.max_x, widget.bounds.max_y);
|
" Bounds: (%.1f, %.1f) → (%.1f, %.1f)\n", widget.bounds.min_x,
|
||||||
|
widget.bounds.min_y, widget.bounds.max_x, widget.bounds.max_y);
|
||||||
|
} else {
|
||||||
|
std::cout << " Bounds: (not available)\n";
|
||||||
|
}
|
||||||
|
std::cout << " Last seen: frame " << widget.last_seen_frame;
|
||||||
|
if (widget.last_seen_at.has_value()) {
|
||||||
|
std::cout << " @ "
|
||||||
|
<< absl::FormatTime("%Y-%m-%d %H:%M:%S",
|
||||||
|
*widget.last_seen_at,
|
||||||
|
absl::LocalTimeZone());
|
||||||
|
}
|
||||||
|
if (widget.stale) {
|
||||||
|
std::cout << " [STALE]";
|
||||||
|
}
|
||||||
|
std::cout << "\n";
|
||||||
std::cout << " Widget ID: 0x" << std::hex << widget.widget_id
|
std::cout << " Widget ID: 0x" << std::hex << widget.widget_id
|
||||||
<< std::dec << "\n";
|
<< std::dec << "\n";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -519,11 +519,19 @@ absl::StatusOr<DiscoverWidgetsResult> GuiAutomationClient::DiscoverWidgets(
|
|||||||
widget.suggested_action = widget_proto.suggested_action();
|
widget.suggested_action = widget_proto.suggested_action();
|
||||||
widget.visible = widget_proto.visible();
|
widget.visible = widget_proto.visible();
|
||||||
widget.enabled = widget_proto.enabled();
|
widget.enabled = widget_proto.enabled();
|
||||||
widget.bounds.min_x = widget_proto.bounds().min_x();
|
widget.has_bounds = widget_proto.has_bounds();
|
||||||
widget.bounds.min_y = widget_proto.bounds().min_y();
|
if (widget.has_bounds) {
|
||||||
widget.bounds.max_x = widget_proto.bounds().max_x();
|
widget.bounds.min_x = widget_proto.bounds().min_x();
|
||||||
widget.bounds.max_y = widget_proto.bounds().max_y();
|
widget.bounds.min_y = widget_proto.bounds().min_y();
|
||||||
|
widget.bounds.max_x = widget_proto.bounds().max_x();
|
||||||
|
widget.bounds.max_y = widget_proto.bounds().max_y();
|
||||||
|
} else {
|
||||||
|
widget.bounds = WidgetBoundingBox();
|
||||||
|
}
|
||||||
widget.widget_id = widget_proto.widget_id();
|
widget.widget_id = widget_proto.widget_id();
|
||||||
|
widget.last_seen_frame = widget_proto.last_seen_frame();
|
||||||
|
widget.last_seen_at = OptionalTimeFromMillis(widget_proto.last_seen_at_ms());
|
||||||
|
widget.stale = widget_proto.stale();
|
||||||
window_info.widgets.push_back(std::move(widget));
|
window_info.widgets.push_back(std::move(widget));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -152,6 +152,10 @@ struct WidgetDescriptor {
|
|||||||
bool enabled = true;
|
bool enabled = true;
|
||||||
WidgetBoundingBox bounds;
|
WidgetBoundingBox bounds;
|
||||||
uint32_t widget_id = 0;
|
uint32_t widget_id = 0;
|
||||||
|
bool has_bounds = false;
|
||||||
|
int64_t last_seen_frame = -1;
|
||||||
|
std::optional<absl::Time> last_seen_at;
|
||||||
|
bool stale = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct DiscoveredWindowInfo {
|
struct DiscoveredWindowInfo {
|
||||||
|
|||||||
Reference in New Issue
Block a user