feat: Enhance widget discovery with telemetry data and improve output formatting
This commit is contained in:
@@ -202,31 +202,48 @@ z3ed gui discover --window "Overworld"
|
||||
z3ed gui discover --type button
|
||||
```
|
||||
|
||||
**API Schema**:
|
||||
**API Schema (current)**:
|
||||
```proto
|
||||
message DiscoverWidgetsRequest {
|
||||
string window_filter = 1; // Optional: filter by window name
|
||||
enum WidgetType { ALL = 0; BUTTON = 1; INPUT = 2; MENU = 3; TAB = 4; CHECKBOX = 5; }
|
||||
string window_filter = 1;
|
||||
WidgetType type_filter = 2;
|
||||
string path_prefix = 3;
|
||||
bool include_invisible = 4;
|
||||
bool include_disabled = 5;
|
||||
}
|
||||
|
||||
message WidgetBounds {
|
||||
float min_x = 1;
|
||||
float min_y = 2;
|
||||
float max_x = 3;
|
||||
float max_y = 4;
|
||||
}
|
||||
|
||||
message DiscoveredWidget {
|
||||
string path = 1;
|
||||
string label = 2;
|
||||
string type = 3;
|
||||
string description = 4;
|
||||
string suggested_action = 5;
|
||||
bool visible = 6;
|
||||
bool enabled = 7;
|
||||
WidgetBounds bounds = 8;
|
||||
uint32 widget_id = 9;
|
||||
int64 last_seen_frame = 10;
|
||||
int64 last_seen_at_ms = 11;
|
||||
bool stale = 12;
|
||||
}
|
||||
|
||||
message DiscoveredWindow {
|
||||
string name = 1;
|
||||
bool visible = 2;
|
||||
repeated DiscoveredWidget widgets = 3;
|
||||
}
|
||||
|
||||
message DiscoverWidgetsResponse {
|
||||
repeated WindowInfo windows = 1;
|
||||
}
|
||||
|
||||
message WindowInfo {
|
||||
string name = 1;
|
||||
bool is_visible = 2;
|
||||
repeated WidgetInfo widgets = 3;
|
||||
}
|
||||
|
||||
message WidgetInfo {
|
||||
string id = 1;
|
||||
string label = 2;
|
||||
string type = 3; // "button", "input", "menu", etc.
|
||||
bool is_enabled = 4;
|
||||
string position = 5; // "x,y,width,height"
|
||||
string suggested_action = 6; // "Click button:Open ROM"
|
||||
repeated DiscoveredWindow windows = 1;
|
||||
int32 total_widgets = 2;
|
||||
int64 generated_at_ms = 3;
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -266,6 +266,12 @@ Options:
|
||||
--format <mode> Output as `table` (default) or `json`
|
||||
--limit <n> Maximum widgets to display (useful for large UIs)
|
||||
|
||||
Each discovered widget now reports:
|
||||
- Current visibility/enablement and bounding box (when available)
|
||||
- Last observed frame number and UTC timestamp
|
||||
- A `stale` flag when the widget hasn't appeared in the current frame
|
||||
- Its underlying ImGui ID for low-level automation
|
||||
|
||||
Examples:
|
||||
# Discover all widgets currently registered
|
||||
z3ed agent gui discover
|
||||
@@ -292,10 +298,13 @@ Window: Overworld (visible)
|
||||
Suggested: Click button:Save
|
||||
State: visible, enabled
|
||||
Bounds: (24.0, 64.0) → (112.0, 92.0)
|
||||
Last seen: frame 18432 @ 2025-01-16 19:42:05
|
||||
Widget ID: 0x13fc41a2
|
||||
|
||||
Widgets shown: 3 of 18 (truncated)
|
||||
Snapshot: 2025-01-16 19:42:05
|
||||
|
||||
_Widgets that have not appeared in the current frame are marked `[STALE]` in the table output._
|
||||
```
|
||||
|
||||
**Use Cases**:
|
||||
|
||||
@@ -121,7 +121,7 @@ z3ed agent test results --test-id grpc_click_12345678 --include-logs
|
||||
z3ed agent test list --category grpc
|
||||
```
|
||||
|
||||
#### Widget Discovery (IT-06) 🔜 PLANNED
|
||||
#### Widget Discovery (IT-06) <EFBFBD> IN PROGRESS — telemetry available
|
||||
```bash
|
||||
# Discover all widgets
|
||||
z3ed agent gui discover
|
||||
@@ -129,8 +129,8 @@ z3ed agent gui discover
|
||||
# Filter by window
|
||||
z3ed agent gui discover --window "Overworld"
|
||||
|
||||
# Get only buttons
|
||||
z3ed agent gui discover --type button --format json
|
||||
# Get only buttons and include hidden/disabled widgets for AI diffing
|
||||
z3ed agent gui discover --type button --include-invisible --include-disabled --format json
|
||||
```
|
||||
|
||||
#### Test Recording (IT-07) 🔜 PLANNED
|
||||
@@ -365,7 +365,7 @@ grpcurl ... Wait '{"condition":"window_visible:WindowName"}'
|
||||
# 3. Assert widget exists
|
||||
grpcurl ... Assert '{"condition":"exists:button:XYZ"}'
|
||||
|
||||
# 4. Use widget discovery (IT-06, planned)
|
||||
# 4. Use widget discovery (IT-06 telemetry)
|
||||
z3ed agent gui discover --window "WindowName"
|
||||
```
|
||||
|
||||
|
||||
Reference in New Issue
Block a user