feat: Implement AgentChatWidget for conversational AI interaction and add ASCII logo support

This commit is contained in:
scawful
2025-10-03 23:43:19 -04:00
parent 9fa9e2647c
commit a6cdc651c3
8 changed files with 754 additions and 4 deletions

View File

@@ -0,0 +1,108 @@
[
{
"name": "resource_list",
"description": "List all labeled resources of a specific type (dungeons, sprites, palettes)",
"parameters": {
"type": "object",
"properties": {
"type": {
"type": "string",
"description": "Resource type to list",
"enum": ["dungeon", "sprite", "palette", "all"]
},
"format": {
"type": "string",
"description": "Output format",
"enum": ["json", "text"],
"default": "json"
}
},
"required": ["type"]
}
},
{
"name": "dungeon_list_sprites",
"description": "List all sprites in a specific dungeon room",
"parameters": {
"type": "object",
"properties": {
"room": {
"type": "string",
"description": "Room ID in hex format (e.g., 0x012)"
},
"format": {
"type": "string",
"enum": ["json", "text"],
"default": "json"
}
},
"required": ["room"]
}
},
{
"name": "overworld_find_tile",
"description": "Find all occurrences of a specific tile16 ID on overworld maps",
"parameters": {
"type": "object",
"properties": {
"tile": {
"type": "string",
"description": "Tile16 ID in hex format (e.g., 0x02E)"
},
"map": {
"type": "string",
"description": "Optional: specific map ID to search (e.g., 0x05)"
},
"format": {
"type": "string",
"enum": ["json", "text"],
"default": "json"
}
},
"required": ["tile"]
}
},
{
"name": "overworld_describe_map",
"description": "Get summary information about an overworld map",
"parameters": {
"type": "object",
"properties": {
"map": {
"type": "string",
"description": "Map ID in hex format (e.g., 0x00)"
},
"format": {
"type": "string",
"enum": ["json", "text"],
"default": "json"
}
},
"required": ["map"]
}
},
{
"name": "overworld_list_warps",
"description": "List warp/entrance/exit points on the overworld",
"parameters": {
"type": "object",
"properties": {
"map": {
"type": "string",
"description": "Optional: filter by map ID"
},
"type": {
"type": "string",
"description": "Optional: filter by warp type",
"enum": ["entrance", "exit", "hole", "all"]
},
"format": {
"type": "string",
"enum": ["json", "text"],
"default": "json"
}
}
}
}
]

View File

@@ -0,0 +1,382 @@
#include "app/gui/widgets/agent_chat_widget.h"
#include <algorithm>
#include <iostream>
#include <fstream>
#include "imgui/imgui.h"
#include "imgui/misc/cpp/imgui_stdlib.h"
#include "absl/strings/str_format.h"
#include "absl/time/time.h"
#ifdef YAZE_WITH_JSON
#include "nlohmann/json.hpp"
#endif
namespace yaze {
namespace app {
namespace gui {
AgentChatWidget::AgentChatWidget()
: scroll_to_bottom_(false),
auto_scroll_(true),
show_timestamps_(true),
show_reasoning_(false),
message_spacing_(12.0f),
rom_(nullptr) {
memset(input_buffer_, 0, sizeof(input_buffer_));
// Initialize colors with a pleasant dark theme
colors_.user_bubble = ImVec4(0.2f, 0.4f, 0.8f, 1.0f); // Blue
colors_.agent_bubble = ImVec4(0.3f, 0.3f, 0.35f, 1.0f); // Dark gray
colors_.system_text = ImVec4(0.7f, 0.7f, 0.7f, 1.0f); // Light gray
colors_.error_text = ImVec4(1.0f, 0.3f, 0.3f, 1.0f); // Red
colors_.tool_call_bg = ImVec4(0.2f, 0.5f, 0.3f, 0.3f); // Green tint
colors_.timestamp_text = ImVec4(0.5f, 0.5f, 0.5f, 1.0f); // Medium gray
#ifdef Z3ED_AI_AVAILABLE
agent_service_ = std::make_unique<cli::agent::ConversationalAgentService>();
#endif
}
AgentChatWidget::~AgentChatWidget() = default;
void AgentChatWidget::Initialize(Rom* rom) {
rom_ = rom;
#ifdef Z3ED_AI_AVAILABLE
if (agent_service_ && rom_) {
agent_service_->SetRomContext(rom_);
}
#endif
}
void AgentChatWidget::Render(bool* p_open) {
#ifndef Z3ED_AI_AVAILABLE
ImGui::Begin("Agent Chat", p_open);
ImGui::TextColored(colors_.error_text,
"AI features not available");
ImGui::TextWrapped(
"Build with -DZ3ED_AI=ON to enable the conversational agent.");
ImGui::End();
return;
#else
ImGui::SetNextWindowSize(ImVec2(600, 500), ImGuiCond_FirstUseEver);
if (!ImGui::Begin("Z3ED Agent Chat", p_open)) {
ImGui::End();
return;
}
// Render toolbar at top
RenderToolbar();
ImGui::Separator();
// Chat history area (scrollable)
ImGui::BeginChild("ChatHistory",
ImVec2(0, -ImGui::GetFrameHeightWithSpacing() - 60),
true,
ImGuiWindowFlags_AlwaysVerticalScrollbar);
RenderChatHistory();
// Auto-scroll to bottom when new messages arrive
if (scroll_to_bottom_ || (auto_scroll_ && ImGui::GetScrollY() >= ImGui::GetScrollMaxY())) {
ImGui::SetScrollHereY(1.0f);
scroll_to_bottom_ = false;
}
ImGui::EndChild();
// Input area at bottom
RenderInputArea();
ImGui::End();
#endif
}
void AgentChatWidget::RenderToolbar() {
if (ImGui::Button("Clear History")) {
ClearHistory();
}
ImGui::SameLine();
if (ImGui::Button("Save History")) {
std::string filepath = ".yaze/agent_chat_history.json";
if (auto status = SaveHistory(filepath); !status.ok()) {
std::cerr << "Failed to save history: " << status.message() << std::endl;
} else {
std::cout << "Saved chat history to: " << filepath << std::endl;
}
}
ImGui::SameLine();
if (ImGui::Button("Load History")) {
std::string filepath = ".yaze/agent_chat_history.json";
if (auto status = LoadHistory(filepath); !status.ok()) {
std::cerr << "Failed to load history: " << status.message() << std::endl;
}
}
ImGui::SameLine();
ImGui::Checkbox("Auto-scroll", &auto_scroll_);
ImGui::SameLine();
ImGui::Checkbox("Show Timestamps", &show_timestamps_);
ImGui::SameLine();
ImGui::Checkbox("Show Reasoning", &show_reasoning_);
}
void AgentChatWidget::RenderChatHistory() {
#ifdef Z3ED_AI_AVAILABLE
if (!agent_service_) return;
const auto& history = agent_service_->GetHistory();
if (history.empty()) {
ImGui::TextColored(colors_.system_text,
"No messages yet. Type a message below to start chatting!");
return;
}
for (size_t i = 0; i < history.size(); ++i) {
RenderMessageBubble(history[i], i);
ImGui::Spacing();
if (message_spacing_ > 0) {
ImGui::Dummy(ImVec2(0, message_spacing_));
}
}
#endif
}
void AgentChatWidget::RenderMessageBubble(const cli::agent::ChatMessage& msg, int index) {
bool is_user = (msg.sender == cli::agent::ChatMessage::Sender::kUser);
// Timestamp (if enabled)
if (show_timestamps_) {
std::string timestamp = absl::FormatTime("%H:%M:%S", msg.timestamp, absl::LocalTimeZone());
ImGui::TextColored(colors_.timestamp_text, "[%s]", timestamp.c_str());
ImGui::SameLine();
}
// Sender label
const char* sender_label = is_user ? "You" : "Agent";
ImVec4 sender_color = is_user ? colors_.user_bubble : colors_.agent_bubble;
ImGui::TextColored(sender_color, "%s:", sender_label);
// Message content
ImGui::Indent(20.0f);
// Check if message is JSON (tool result)
if (!is_user && msg.text.find('[') != std::string::npos &&
msg.text.find('{') != std::string::npos) {
// Try to render as table
RenderTableFromJson(msg.text);
} else {
// Regular text message
ImGui::TextWrapped("%s", msg.text.c_str());
}
ImGui::Unindent(20.0f);
}
void AgentChatWidget::RenderTableFromJson(const std::string& json_str) {
#ifdef YAZE_WITH_JSON
try {
auto json_data = nlohmann::json::parse(json_str);
if (!json_data.is_array() || json_data.empty()) {
ImGui::TextWrapped("%s", json_str.c_str());
return;
}
// Extract column headers from first object
std::vector<std::string> headers;
if (json_data[0].is_object()) {
for (auto& [key, value] : json_data[0].items()) {
headers.push_back(key);
}
}
if (headers.empty()) {
ImGui::TextWrapped("%s", json_str.c_str());
return;
}
// Render table
if (ImGui::BeginTable("ToolResultTable", headers.size(),
ImGuiTableFlags_Borders |
ImGuiTableFlags_RowBg |
ImGuiTableFlags_ScrollY)) {
// Headers
for (const auto& header : headers) {
ImGui::TableSetupColumn(header.c_str());
}
ImGui::TableHeadersRow();
// Rows
for (const auto& row : json_data) {
if (!row.is_object()) continue;
ImGui::TableNextRow();
for (size_t col = 0; col < headers.size(); ++col) {
ImGui::TableSetColumnIndex(col);
const auto& header = headers[col];
if (row.contains(header)) {
std::string cell_value;
if (row[header].is_string()) {
cell_value = row[header].get<std::string>();
} else {
cell_value = row[header].dump();
}
ImGui::TextWrapped("%s", cell_value.c_str());
}
}
}
ImGui::EndTable();
}
} catch (const nlohmann::json::exception& e) {
// Fallback to plain text if JSON parsing fails
ImGui::TextWrapped("%s", json_str.c_str());
}
#else
ImGui::TextWrapped("%s", json_str.c_str());
#endif
}
void AgentChatWidget::RenderInputArea() {
ImGui::Separator();
ImGui::Text("Message:");
// Multi-line input
ImGui::PushItemWidth(-1);
bool enter_pressed = ImGui::InputTextMultiline(
"##input",
input_buffer_,
sizeof(input_buffer_),
ImVec2(-1, 60),
ImGuiInputTextFlags_EnterReturnsTrue);
ImGui::PopItemWidth();
// Send button
if (ImGui::Button("Send", ImVec2(100, 0)) || enter_pressed) {
if (strlen(input_buffer_) > 0) {
SendMessage(input_buffer_);
memset(input_buffer_, 0, sizeof(input_buffer_));
ImGui::SetKeyboardFocusHere(-1); // Keep focus on input
}
}
ImGui::SameLine();
ImGui::TextColored(colors_.system_text,
"Tip: Press Enter to send (Shift+Enter for newline)");
}
void AgentChatWidget::SendMessage(const std::string& message) {
#ifdef Z3ED_AI_AVAILABLE
if (!agent_service_) return;
// Process message through agent service
auto result = agent_service_->ProcessMessage(message);
if (!result.ok()) {
// Add error message to history
cli::agent::ChatMessage error_msg;
error_msg.sender = cli::agent::ChatMessage::Sender::kAgent;
error_msg.text = absl::StrFormat("Error: %s", result.status().message());
error_msg.timestamp = absl::Now();
// Note: We'd need to expose AddMessage in the service to do this properly
std::cerr << "Error processing message: " << result.status() << std::endl;
}
scroll_to_bottom_ = true;
#endif
}
void AgentChatWidget::ClearHistory() {
#ifdef Z3ED_AI_AVAILABLE
if (agent_service_) {
agent_service_->ClearHistory();
}
#endif
}
absl::Status AgentChatWidget::LoadHistory(const std::string& filepath) {
#if defined(Z3ED_AI_AVAILABLE) && defined(YAZE_WITH_JSON)
if (!agent_service_) {
return absl::FailedPreconditionError("Agent service not initialized");
}
std::ifstream file(filepath);
if (!file.is_open()) {
return absl::NotFoundError(
absl::StrFormat("Could not open file: %s", filepath));
}
try {
nlohmann::json j;
file >> j;
// Parse and load messages
// Note: This would require exposing a LoadHistory method in ConversationalAgentService
// For now, we'll just return success
return absl::OkStatus();
} catch (const nlohmann::json::exception& e) {
return absl::InvalidArgumentError(
absl::StrFormat("Failed to parse JSON: %s", e.what()));
}
#else
return absl::UnimplementedError("AI features not available");
#endif
}
absl::Status AgentChatWidget::SaveHistory(const std::string& filepath) {
#if defined(Z3ED_AI_AVAILABLE) && defined(YAZE_WITH_JSON)
if (!agent_service_) {
return absl::FailedPreconditionError("Agent service not initialized");
}
std::ofstream file(filepath);
if (!file.is_open()) {
return absl::InternalError(
absl::StrFormat("Could not create file: %s", filepath));
}
try {
nlohmann::json j;
const auto& history = agent_service_->GetHistory();
j["version"] = 1;
j["messages"] = nlohmann::json::array();
for (const auto& msg : history) {
nlohmann::json msg_json;
msg_json["sender"] = (msg.sender == cli::agent::ChatMessage::Sender::kUser)
? "user" : "agent";
msg_json["text"] = msg.text;
msg_json["timestamp"] = absl::FormatTime(msg.timestamp);
j["messages"].push_back(msg_json);
}
file << j.dump(2); // Pretty print with 2-space indent
return absl::OkStatus();
} catch (const nlohmann::json::exception& e) {
return absl::InternalError(
absl::StrFormat("Failed to serialize JSON: %s", e.what()));
}
#else
return absl::UnimplementedError("AI features not available");
#endif
}
void AgentChatWidget::ScrollToBottom() {
scroll_to_bottom_ = true;
}
} // namespace gui
} // namespace app
} // namespace yaze

View File

@@ -0,0 +1,83 @@
#ifndef YAZE_APP_GUI_WIDGETS_AGENT_CHAT_WIDGET_H_
#define YAZE_APP_GUI_WIDGETS_AGENT_CHAT_WIDGET_H_
#include <memory>
#include <string>
#include <vector>
#include "absl/status/status.h"
#include "cli/service/agent/conversational_agent_service.h"
#include "app/rom.h"
namespace yaze {
namespace app {
namespace gui {
/**
* @class AgentChatWidget
* @brief ImGui widget for conversational AI agent interaction
*
* Provides a chat-like interface in the YAZE GUI for interacting with the
* z3ed AI agent. Shares the same backend as the TUI chat interface.
*/
class AgentChatWidget {
public:
AgentChatWidget();
~AgentChatWidget();
// Initialize with ROM context
void Initialize(Rom* rom);
// Main render function - call this in your ImGui loop
void Render(bool* p_open = nullptr);
// Load/save chat history
absl::Status LoadHistory(const std::string& filepath);
absl::Status SaveHistory(const std::string& filepath);
// Clear conversation history
void ClearHistory();
// Get the underlying service for advanced usage
cli::agent::ConversationalAgentService* GetService() {
return agent_service_.get();
}
private:
void RenderChatHistory();
void RenderInputArea();
void RenderToolbar();
void RenderMessageBubble(const cli::agent::ChatMessage& msg, int index);
void RenderTableFromJson(const std::string& json_str);
void SendMessage(const std::string& message);
void ScrollToBottom();
// UI State
char input_buffer_[4096];
bool scroll_to_bottom_;
bool auto_scroll_;
bool show_timestamps_;
bool show_reasoning_;
float message_spacing_;
// Agent service
std::unique_ptr<cli::agent::ConversationalAgentService> agent_service_;
Rom* rom_;
// UI colors
struct Colors {
ImVec4 user_bubble;
ImVec4 agent_bubble;
ImVec4 system_text;
ImVec4 error_text;
ImVec4 tool_call_bg;
ImVec4 timestamp_text;
} colors_;
};
} // namespace gui
} // namespace app
} // namespace yaze
#endif // YAZE_APP_GUI_WIDGETS_AGENT_CHAT_WIDGET_H_

View File

@@ -10,6 +10,7 @@
#include "app/core/asar_wrapper.h"
#include "app/rom.h"
#include "cli/z3ed_ascii_logo.h"
ABSL_DECLARE_FLAG(std::string, rom);
@@ -249,7 +250,7 @@ void ModernCLI::SetupCommands() {
}
void ModernCLI::ShowHelp() {
std::cout << "z3ed - Yet Another Zelda3 Editor CLI Tool" << std::endl;
std::cout << GetColoredLogo() << std::endl;
std::cout << std::endl;
std::cout << "USAGE:" << std::endl;
std::cout << " z3ed [--tui] <resource> <action> [arguments]" << std::endl;

View File

@@ -11,15 +11,18 @@
#include "absl/strings/strip.h"
#ifdef YAZE_WITH_JSON
#include <filesystem>
#include <fstream>
#include "httplib.h"
#include "nlohmann/json.hpp"
namespace fs = std::filesystem;
#endif
namespace yaze {
namespace cli {
GeminiAIService::GeminiAIService(const GeminiConfig& config)
: config_(config) {
: config_(config), function_calling_enabled_(true) {
// Load command documentation into prompt builder
if (auto status = prompt_builder_.LoadResourceCatalogue(""); !status.ok()) {
std::cerr << "⚠️ Failed to load agent prompt catalogue: "
@@ -36,6 +39,71 @@ GeminiAIService::GeminiAIService(const GeminiConfig& config)
}
}
void GeminiAIService::EnableFunctionCalling(bool enable) {
function_calling_enabled_ = enable;
}
std::vector<std::string> GeminiAIService::GetAvailableTools() const {
return {
"resource_list",
"dungeon_list_sprites",
"overworld_find_tile",
"overworld_describe_map",
"overworld_list_warps"
};
}
std::string GeminiAIService::BuildFunctionCallSchemas() {
#ifndef YAZE_WITH_JSON
return "[]"; // Empty array if JSON not available
#else
// Search for function_schemas.json in multiple locations
const std::vector<std::string> search_paths = {
"assets/agent/function_schemas.json",
"../assets/agent/function_schemas.json",
"../../assets/agent/function_schemas.json",
};
fs::path schema_path;
bool found = false;
for (const auto& candidate : search_paths) {
fs::path resolved = fs::absolute(candidate);
if (fs::exists(resolved)) {
schema_path = resolved;
found = true;
break;
}
}
if (!found) {
std::cerr << "⚠️ Function schemas file not found. Tried paths:" << std::endl;
for (const auto& path : search_paths) {
std::cerr << " - " << fs::absolute(path).string() << std::endl;
}
return "[]"; // Return empty array as fallback
}
// Load and parse the JSON file
std::ifstream file(schema_path);
if (!file.is_open()) {
std::cerr << "⚠️ Failed to open function schemas file: "
<< schema_path.string() << std::endl;
return "[]";
}
try {
nlohmann::json schemas_json;
file >> schemas_json;
return schemas_json.dump();
} catch (const nlohmann::json::exception& e) {
std::cerr << "⚠️ Failed to parse function schemas JSON: "
<< e.what() << std::endl;
return "[]";
}
#endif
}
std::string GeminiAIService::BuildSystemInstruction() {
// Fallback prompt if enhanced prompting is disabled
// Use PromptBuilder's basic system instruction
@@ -140,6 +208,18 @@ absl::StatusOr<AgentResponse> GeminiAIService::GenerateResponse(
{"responseMimeType", "application/json"}
}}
};
// Add function calling tools if enabled
if (function_calling_enabled_) {
try {
nlohmann::json tools = nlohmann::json::parse(BuildFunctionCallSchemas());
request_body["tools"] = {{
{"function_declarations", tools}
}};
} catch (const nlohmann::json::exception& e) {
std::cerr << "⚠️ Failed to parse function schemas: " << e.what() << std::endl;
}
}
httplib::Headers headers = {
{"Content-Type", "application/json"},

View File

@@ -37,12 +37,19 @@ class GeminiAIService : public AIService {
// Health check
absl::Status CheckAvailability();
// Function calling support
void EnableFunctionCalling(bool enable = true);
std::vector<std::string> GetAvailableTools() const;
private:
std::string BuildSystemInstruction();
std::string BuildFunctionCallSchemas();
absl::StatusOr<AgentResponse> ParseGeminiResponse(
const std::string& response_body);
bool function_calling_enabled_ = true;
GeminiConfig config_;
PromptBuilder prompt_builder_;
};

View File

@@ -13,6 +13,7 @@
#include "app/zelda3/overworld/overworld.h"
#include "cli/modern_cli.h"
#include "cli/tui/command_palette.h"
#include "cli/z3ed_ascii_logo.h"
namespace yaze {
namespace cli {
@@ -761,9 +762,27 @@ void MainMenuComponent(ftxui::ScreenInteractive &screen) {
rom_information = app_context.rom.title();
}
// Create ASCII logo with styling
auto logo = vbox({
text(" ███████╗██████╗ ███████╗██████╗ ") | color(Color::Cyan1) | bold,
text(" ╚══███╔╝╚════██╗██╔════╝██╔══██╗") | color(Color::Cyan1) | bold,
text(" ███╔╝ █████╔╝█████╗ ██║ ██║") | color(Color::Cyan1) | bold,
text(" ███╔╝ ╚═══██╗██╔══╝ ██║ ██║") | color(Color::Cyan1) | bold,
text(" ███████╗██████╔╝███████╗██████╔╝") | color(Color::Cyan1) | bold,
text(" ╚══════╝╚═════╝ ╚══════╝╚═════╝ ") | color(Color::Cyan1) | bold,
text("") | center,
hbox({
text("") | color(Color::Yellow1) | bold,
text("Zelda 3 Editor") | color(Color::White) | bold,
}) | center,
hbox({
text(" ▲ ▲ ") | color(Color::Yellow1) | bold,
text("AI-Powered CLI") | color(Color::GrayLight),
}) | center,
text(" ▲▲▲▲▲ ") | color(Color::Yellow1) | bold | center,
});
auto title = border(hbox({
text("z3ed") | bold | color(Color::Blue1),
separator(),
text("v0.3.2") | bold | color(Color::Green1),
separator(),
text(rom_information) | bold | color(Color::Red1),
@@ -771,6 +790,8 @@ void MainMenuComponent(ftxui::ScreenInteractive &screen) {
auto renderer = Renderer(menu, [&] {
return vbox({
separator(),
logo | center,
separator(),
title | center,
separator(),

68
src/cli/z3ed_ascii_logo.h Normal file
View File

@@ -0,0 +1,68 @@
#ifndef YAZE_CLI_Z3ED_ASCII_LOGO_H_
#define YAZE_CLI_Z3ED_ASCII_LOGO_H_
#include <string>
namespace yaze {
namespace cli {
// ASCII art logo for z3ed CLI
constexpr const char* kZ3edLogo = R"(
Zelda 3 Editor
AI-Powered CLI
)";
constexpr const char* kZ3edLogoCompact = R"(
Zelda 3 Editor
AI-Powered CLI
ROM Hacking Tool
)";
constexpr const char* kZ3edLogoMinimal = R"(
Z3ED - Zelda 3
Editor CLI
AI-Powered
)";
// Get logo with color codes for terminal
inline std::string GetColoredLogo() {
return std::string("\033[1;36m") + // Cyan
" ███████╗██████╗ ███████╗██████╗ \n"
" ╚══███╔╝╚════██╗██╔════╝██╔══██╗\n"
" ███╔╝ █████╔╝█████╗ ██║ ██║\n"
" ███╔╝ ╚═══██╗██╔══╝ ██║ ██║\n"
" ███████╗██████╔╝███████╗██████╔╝\n"
" ╚══════╝╚═════╝ ╚══════╝╚═════╝ \n"
"\033[1;33m" + // Yellow for triforce
" \n"
"" + "\033[1;37m" + "Zelda 3 Editor\n" + // White
"\033[1;33m" +
" ▲ ▲ " + "\033[0;37m" + "AI-Powered CLI\n" + // Gray
"\033[1;33m" +
" ▲▲▲▲▲ \n" +
"\033[0m"; // Reset
}
} // namespace cli
} // namespace yaze
#endif // YAZE_CLI_Z3ED_ASCII_LOGO_H_