From 21ef67c1ca59c6a09ce53c371c778646f5884e03 Mon Sep 17 00:00:00 2001 From: scawful Date: Fri, 3 Oct 2025 15:44:11 -0400 Subject: [PATCH] Update CMake presets for agent testing and enhance roadmap documentation --- CMakePresets.json | 78 +++++++++++++++++++++++++++++++++----- docs/z3ed/AGENT-ROADMAP.md | 38 ++++++++----------- src/lib/imgui | 2 +- 3 files changed, 84 insertions(+), 34 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index f5a62a61..806242e0 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -17,7 +17,7 @@ "YAZE_BUILD_TESTS": "ON", "YAZE_BUILD_APP": "ON", "YAZE_BUILD_LIB": "ON", - "YAZE_BUILD_EMU": "ON", + "YAZE_BUILD_EMU": "OFF", "YAZE_BUILD_Z3ED": "ON" } }, @@ -50,17 +50,44 @@ "inherits": "debug", "cacheVariables": { "YAZE_ENABLE_ROM_TESTS": "ON", - "YAZE_TEST_ROM_PATH": "${sourceDir}/build/bin/zelda3.sfc" + "YAZE_TEST_ROM_PATH": "${sourceDir}/zelda3.sfc" } }, { "name": "macos-dev", - "displayName": "macOS Development (ARM64)", + "displayName": "macOS debug (ARM64)", "description": "macOS ARM64 development build with ROM testing", "inherits": "macos-debug", "cacheVariables": { "YAZE_ENABLE_ROM_TESTS": "ON", - "YAZE_TEST_ROM_PATH": "${sourceDir}/build/bin/zelda3.sfc" + "YAZE_TEST_ROM_PATH": "${sourceDir}/zelda3.sfc" + } + }, + { + "name": "macos-dev-z3ed-ai", + "displayName": "macOS debug z3ed agent (ARM64)", + "description": "macOS ARM64 development build with ROM testing", + "inherits": "macos-debug", + "cacheVariables": { + "YAZE_ENABLE_ROM_TESTS": "ON", + "YAZE_TEST_ROM_PATH": "${sourceDir}/zelda3.sfc", + "YAZE_WITH_JSON": "ON", + "YAZE_WITH_GRPC": "ON", + "YAZE_BUILD_Z3ED": "ON" + } + }, + { + "name": "macos-agent-test", + "displayName": "macOS z3ed agent test (ARM64)", + "description": "macOS ARM64 build for exercising the z3ed agent with JSON/GRPC", + "inherits": "macos-debug", + "cacheVariables": { + "YAZE_WITH_JSON": "ON", + "YAZE_WITH_GRPC": "ON", + "YAZE_BUILD_Z3ED": "ON", + "YAZE_BUILD_TESTS": "ON", + "YAZE_ENABLE_ROM_TESTS": "ON", + "YAZE_TEST_ROM_PATH": "${sourceDir}/zelda3.sfc" } }, { @@ -219,6 +246,20 @@ "YAZE_ENABLE_ROM_TESTS": "ON" } }, + { + "name": "windows-agent-test", + "displayName": "Windows z3ed agent test", + "description": "Windows build for exercising the z3ed agent with JSON/GRPC", + "inherits": "windows-debug", + "cacheVariables": { + "YAZE_WITH_JSON": "ON", + "YAZE_WITH_GRPC": "ON", + "YAZE_BUILD_Z3ED": "ON", + "YAZE_BUILD_TESTS": "ON", + "YAZE_ENABLE_ROM_TESTS": "ON", + "YAZE_TEST_ROM_PATH": "${sourceDir}/zelda3.sfc" + } + }, { "name": "windows-arm64-debug", "displayName": "Windows ARM64 Debug", @@ -350,6 +391,11 @@ "configurePreset": "macos-release-universal", "displayName": "macOS Release Build (Universal)" }, + { + "name": "macos-agent-test", + "configurePreset": "macos-agent-test", + "displayName": "macOS z3ed Agent Test Build" + }, { "name": "fast", "configurePreset": "debug", @@ -359,32 +405,44 @@ { "name": "windows-debug", "configurePreset": "windows-debug", - "displayName": "Windows Debug Build" + "displayName": "Windows Debug Build", + "configuration": "Debug" }, { "name": "windows-release", "configurePreset": "windows-release", - "displayName": "Windows Release Build" + "displayName": "Windows Release Build", + "configuration": "Release" }, { "name": "windows-dev", "configurePreset": "windows-dev", - "displayName": "Windows Development Build" + "displayName": "Windows Development Build", + "configuration": "Debug" }, { "name": "windows-arm64-debug", "configurePreset": "windows-arm64-debug", - "displayName": "Windows ARM64 Debug Build" + "displayName": "Windows ARM64 Debug Build", + "configuration": "Debug" }, { "name": "windows-arm64-release", "configurePreset": "windows-arm64-release", - "displayName": "Windows ARM64 Release Build" + "displayName": "Windows ARM64 Release Build", + "configuration": "Release" }, { "name": "windows-arm64-dev", "configurePreset": "windows-arm64-dev", - "displayName": "Windows ARM64 Development Build" + "displayName": "Windows ARM64 Development Build", + "configuration": "Debug" + }, + { + "name": "windows-agent-test", + "configurePreset": "windows-agent-test", + "displayName": "Windows z3ed Agent Test Build", + "configuration": "Debug" } ], "testPresets": [ diff --git a/docs/z3ed/AGENT-ROADMAP.md b/docs/z3ed/AGENT-ROADMAP.md index 0329d726..0aa28998 100644 --- a/docs/z3ed/AGENT-ROADMAP.md +++ b/docs/z3ed/AGENT-ROADMAP.md @@ -1,7 +1,5 @@ # z3ed Agent Roadmap -**Latest Update**: October 3, 2025 - This document outlines the strategic vision and concrete next steps for the `z3ed` AI agent, focusing on a transition from a command-line tool to a fully interactive, conversational assistant for ROM hacking. ## Core Vision: The Conversational ROM Hacking Assistant @@ -57,44 +55,38 @@ This vision will be realized through a shared interface available in both the `z 6. User uses the `agent diff` and `agent accept` commands (or UI equivalents) to review and apply the changes. - **Status**: The proposal workflow itself is mostly implemented. This task involves integrating it with the new conversational service. -## Consolidated Next Steps +## Next Steps -### Immediate Priorities (Next Session) -1. **Share ROM Context with the Agent**: - - Inject the active GUI/TUI ROM instance into `ConversationalAgentService`. - - Ensure tool calls succeed without requiring `--rom` flags when running inside the editor. -2. **Expand Overworld Tool Coverage**: +### Immediate Priorities +1. **Expand Overworld Tool Coverage**: - Add read-only commands for tile searches, area summaries, and teleport destinations. - Guarantee each tool returns both JSON and human-readable summaries for the chat renderers. -3. **Document & Test the New Tooling**: +2. **Document & Test the New Tooling**: - Update the main `README.md` and relevant docs to cover the new chat formatting. - Add regression tests (unit or golden JSON fixtures) for the new Overworld tools. -### Short-Term Goals (This Week) -1. **Polish the TUI Chat Experience**: +3. **Polish the TUI Chat Experience**: - Tighten keyboard shortcuts, scrolling, and copy-to-clipboard behaviour. - Align log file output with on-screen formatting for easier debugging. -2. **Integrate Tool Use with LLM**: +4. **Integrate Tool Use with LLM**: - Modify the `AIService` to support function calling/tool use. - Teach the agent to call the new read-only commands to answer questions. -3. **Land Overworld Tooling**: +5. **Land Overworld Tooling**: - Ship at least two Overworld inspection commands with comprehensive tests. - - Record example transcripts demonstrating tool usage in both TUI and GUI. -### Long-Term Vision (Next Week and Beyond) -1. **Build GUI Chat Widget**: +6. **Build GUI Chat Widget**: - Create the ImGui component. - Ensure it shares the same backend service as the TUI. -2. **Full Integration with Proposal System**: +7. **Full Integration with Proposal System**: - Implement the logic for the agent to transition from conversation to proposal generation. -3. **Expand Tool Arsenal**: +8. **Expand Tool Arsenal**: - Continuously add new read-only commands to give the agent more capabilities to inspect the ROM. -4. **Multi-Modal Agent**: +9. **Multi-Modal Agent**: - Explore the possibility of the agent generating and displaying images (e.g., a map of a dungeon room) in the chat. -5. **Advanced Configuration**: +10. **Advanced Configuration**: - Implement environment variables for selecting AI providers and models (e.g., `YAZE_AI_PROVIDER`, `OLLAMA_MODEL`). - Add CLI flags for overriding the provider and model on a per-command basis. -6. **Performance and Cost-Saving**: +11. **Performance and Cost-Saving**: - Implement a response cache to reduce latency and API costs. - Add token usage tracking and reporting. @@ -110,11 +102,11 @@ We have made significant progress in laying the foundation for the conversationa - **Tool Use Foundation**: The `ToolDispatcher` is implemented, and the AI services are aware of the new tool call format. - **Tool Loop Improvements**: Conversational flow now handles multi-step tool calls with default JSON output, allowing results to feed back into the chat without recursion. - **Structured Tool Output Rendering**: Both the TUI and GUI chat widgets now display tables and JSON payloads with friendly formatting, drastically improving readability. +- **Overworld Inspection Suite**: Added `overworld describe-map` and `overworld list-warps` commands producing text/JSON summaries for map metadata and warp points, with agent tooling hooks. ### ✅ Build Configuration Issue Resolved The linker error is fixed. Both the CLI and GUI targets now link against `yaze_agent`, so the shared agent handlers (`HandleResourceListCommand`, `HandleDungeonListSpritesCommand`, etc.) compile once and are available to `ToolDispatcher` everywhere. ### 🚀 Next Steps 1. **Share ROM Context with the Agent**: Inject the active GUI ROM into `ConversationalAgentService` so tool calls work even when `--rom` flags are unavailable. Analyze the `src/app/rom.cc` and `src/app/rom.h` and `src/app/editor/editor_manager.cc` files for guidance on accessing the current project/ROM. -2. **Expand Tool Coverage**: Target Overworld navigation helpers (`overworld find-tile`, `overworld list-warps`, region summaries) and dialogue inspectors. Prioritize commands that unblock common level-design questions and emit concise table/JSON payloads. -3. **Publish Usage Examples**: Capture transcripts and screenshots that highlight structured chat output, then weave them into the docs so contributors can see the intended UX. \ No newline at end of file +2. **Expand Tool Coverage**: Target Overworld navigation helpers (`overworld find-tile`, `overworld list-warps`, region summaries) and dialogue inspectors. Prioritize commands that unblock common level-design questions and emit concise table/JSON payloads. \ No newline at end of file diff --git a/src/lib/imgui b/src/lib/imgui index a0a6639f..8f3f4282 160000 --- a/src/lib/imgui +++ b/src/lib/imgui @@ -1 +1 @@ -Subproject commit a0a6639fb5fe8018c2c4b421c8c94243c245ca86 +Subproject commit 8f3f4282288b17aee798b6fade6787a8a17843ca