feat: Enhance widget discovery with telemetry data and improve output formatting

This commit is contained in:
scawful
2025-10-02 22:50:47 -04:00
parent 21074f6445
commit c202aa9fa4
11 changed files with 457 additions and 78 deletions

View File

@@ -202,31 +202,48 @@ z3ed gui discover --window "Overworld"
z3ed gui discover --type button
```
**API Schema**:
**API Schema (current)**:
```proto
message DiscoverWidgetsRequest {
string window_filter = 1; // Optional: filter by window name
enum WidgetType { ALL = 0; BUTTON = 1; INPUT = 2; MENU = 3; TAB = 4; CHECKBOX = 5; }
string window_filter = 1;
WidgetType type_filter = 2;
string path_prefix = 3;
bool include_invisible = 4;
bool include_disabled = 5;
}
message WidgetBounds {
float min_x = 1;
float min_y = 2;
float max_x = 3;
float max_y = 4;
}
message DiscoveredWidget {
string path = 1;
string label = 2;
string type = 3;
string description = 4;
string suggested_action = 5;
bool visible = 6;
bool enabled = 7;
WidgetBounds bounds = 8;
uint32 widget_id = 9;
int64 last_seen_frame = 10;
int64 last_seen_at_ms = 11;
bool stale = 12;
}
message DiscoveredWindow {
string name = 1;
bool visible = 2;
repeated DiscoveredWidget widgets = 3;
}
message DiscoverWidgetsResponse {
repeated WindowInfo windows = 1;
}
message WindowInfo {
string name = 1;
bool is_visible = 2;
repeated WidgetInfo widgets = 3;
}
message WidgetInfo {
string id = 1;
string label = 2;
string type = 3; // "button", "input", "menu", etc.
bool is_enabled = 4;
string position = 5; // "x,y,width,height"
string suggested_action = 6; // "Click button:Open ROM"
repeated DiscoveredWindow windows = 1;
int32 total_widgets = 2;
int64 generated_at_ms = 3;
}
```

View File

@@ -266,6 +266,12 @@ Options:
--format <mode> Output as `table` (default) or `json`
--limit <n> Maximum widgets to display (useful for large UIs)
Each discovered widget now reports:
- Current visibility/enablement and bounding box (when available)
- Last observed frame number and UTC timestamp
- A `stale` flag when the widget hasn't appeared in the current frame
- Its underlying ImGui ID for low-level automation
Examples:
# Discover all widgets currently registered
z3ed agent gui discover
@@ -292,10 +298,13 @@ Window: Overworld (visible)
Suggested: Click button:Save
State: visible, enabled
Bounds: (24.0, 64.0) → (112.0, 92.0)
Last seen: frame 18432 @ 2025-01-16 19:42:05
Widget ID: 0x13fc41a2
Widgets shown: 3 of 18 (truncated)
Snapshot: 2025-01-16 19:42:05
_Widgets that have not appeared in the current frame are marked `[STALE]` in the table output._
```
**Use Cases**:

View File

@@ -121,7 +121,7 @@ z3ed agent test results --test-id grpc_click_12345678 --include-logs
z3ed agent test list --category grpc
```
#### Widget Discovery (IT-06) 🔜 PLANNED
#### Widget Discovery (IT-06) <EFBFBD> IN PROGRESS — telemetry available
```bash
# Discover all widgets
z3ed agent gui discover
@@ -129,8 +129,8 @@ z3ed agent gui discover
# Filter by window
z3ed agent gui discover --window "Overworld"
# Get only buttons
z3ed agent gui discover --type button --format json
# Get only buttons and include hidden/disabled widgets for AI diffing
z3ed agent gui discover --type button --include-invisible --include-disabled --format json
```
#### Test Recording (IT-07) 🔜 PLANNED
@@ -365,7 +365,7 @@ grpcurl ... Wait '{"condition":"window_visible:WindowName"}'
# 3. Assert widget exists
grpcurl ... Assert '{"condition":"exists:button:XYZ"}'
# 4. Use widget discovery (IT-06, planned)
# 4. Use widget discovery (IT-06 telemetry)
z3ed agent gui discover --window "WindowName"
```