scawful/yaze
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement collaboration and multimodal functionality in AgentChatWidget

Browse Source
This commit is contained in:
scawful
2025-10-04 14:14:26 -04:00
parent e840d71add
commit 820e4577df
2 changed files with 417 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

356
src/app/editor/system/agent_chat_widget.cc
View File

@@ -181,6 +181,18 @@ void AgentChatWidget::SetRomContext(Rom* rom) {
agent_service_.SetRomContext(rom);
}
void AgentChatWidget::SetToastManager(ToastManager* toast_manager) {
toast_manager_ = toast_manager;
}
void AgentChatWidget::SetProposalDrawer(ProposalDrawer* drawer) {
proposal_drawer_ = drawer;
if (proposal_drawer_ && !pending_focus_proposal_id_.empty()) {
proposal_drawer_->FocusProposal(pending_focus_proposal_id_);
pending_focus_proposal_id_.clear();
}
}
void AgentChatWidget::EnsureHistoryLoaded() {
if (history_loaded_) {
return;
@@ -272,6 +284,43 @@ void AgentChatWidget::EnsureHistoryLoaded() {
ToastType::kInfo, 3.5f);
}
}
if (json.contains("collaboration") && json["collaboration"].is_object()) {
const auto& collab_json = json["collaboration"];
collaboration_state_.active = collab_json.value("active", false);
collaboration_state_.session_id = collab_json.value("session_id", "");
collaboration_state_.participants.clear();
if (collab_json.contains("participants") &&
collab_json["participants"].is_array()) {
for (const auto& participant : collab_json["participants"]) {
if (participant.is_string()) {
collaboration_state_.participants.push_back(
participant.get<std::string>());
}
}
}
if (collab_json.contains("last_synced")) {
collaboration_state_.last_synced =
ParseTimestamp(collab_json["last_synced"]);
}
}
if (json.contains("multimodal") && json["multimodal"].is_object()) {
const auto& multimodal_json = json["multimodal"];
if (multimodal_json.contains("last_capture_path") &&
multimodal_json["last_capture_path"].is_string()) {
std::string path = multimodal_json["last_capture_path"].get<std::string>();
if (!path.empty()) {
multimodal_state_.last_capture_path = std::filesystem::path(path);
}
}
multimodal_state_.status_message =
multimodal_json.value("status_message", "");
if (multimodal_json.contains("last_updated")) {
multimodal_state_.last_updated =
ParseTimestamp(multimodal_json["last_updated"]);
}
}
} catch (const std::exception& e) {
if (toast_manager_) {
toast_manager_->Show(
@@ -328,6 +377,32 @@ void AgentChatWidget::PersistHistory() {
json["messages"].push_back(std::move(entry));
}
nlohmann::json collab_json;
collab_json["active"] = collaboration_state_.active;
collab_json["session_id"] = collaboration_state_.session_id;
collab_json["participants"] = collaboration_state_.participants;
if (collaboration_state_.last_synced != absl::InfinitePast()) {
collab_json["last_synced"] = absl::FormatTime(
absl::RFC3339_full, collaboration_state_.last_synced,
absl::UTCTimeZone());
}
json["collaboration"] = std::move(collab_json);
nlohmann::json multimodal_json;
if (multimodal_state_.last_capture_path.has_value()) {
multimodal_json["last_capture_path"] =
multimodal_state_.last_capture_path->string();
} else {
multimodal_json["last_capture_path"] = "";
}
multimodal_json["status_message"] = multimodal_state_.status_message;
if (multimodal_state_.last_updated != absl::InfinitePast()) {
multimodal_json["last_updated"] = absl::FormatTime(
absl::RFC3339_full, multimodal_state_.last_updated,
absl::UTCTimeZone());
}
json["multimodal"] = std::move(multimodal_json);
std::error_code ec;
auto directory = history_path_.parent_path();
if (!directory.empty()) {
@@ -521,6 +596,7 @@ void AgentChatWidget::RenderProposalQuickActions(const ChatMessage& msg,
void AgentChatWidget::RenderHistory() {
const auto& history = agent_service_.GetHistory();
float reserved_height = ImGui::GetFrameHeightWithSpacing() * 4.0f;
reserved_height += 220.0f;
if (ImGui::BeginChild("History",
ImVec2(0, -reserved_height),
@@ -591,9 +667,289 @@ void AgentChatWidget::Draw() {
ImGui::Begin(title_.c_str(), &active_);
RenderHistory();
RenderCollaborationPanel();
RenderMultimodalPanel();
RenderInputBox();
ImGui::End();
}
void AgentChatWidget::RenderCollaborationPanel() {
if (!ImGui::CollapsingHeader("Collaborative Session (Preview)",
ImGuiTreeNodeFlags_DefaultOpen)) {
return;
}
const bool connected = collaboration_state_.active;
ImGui::Text("Status: %s", connected ? "Connected" : "Not connected");
if (!collaboration_state_.session_id.empty()) {
ImGui::Text("Session ID: %s", collaboration_state_.session_id.c_str());
}
if (collaboration_state_.last_synced != absl::InfinitePast()) {
ImGui::TextDisabled(
"Last sync: %s",
absl::FormatTime("%H:%M:%S", collaboration_state_.last_synced,
absl::LocalTimeZone()).c_str());
}
ImGui::Separator();
bool can_host = static_cast<bool>(collaboration_callbacks_.host_session);
bool can_join = static_cast<bool>(collaboration_callbacks_.join_session);
bool can_leave = static_cast<bool>(collaboration_callbacks_.leave_session);
ImGui::InputTextWithHint("##session_name", "Session name",
session_name_buffer_,
IM_ARRAYSIZE(session_name_buffer_));
ImGui::SameLine();
if (!can_host) ImGui::BeginDisabled();
if (ImGui::Button("Host Session")) {
std::string name = session_name_buffer_;
if (name.empty()) {
if (toast_manager_) {
toast_manager_->Show("Enter a session name first",
ToastType::kWarning, 3.0f);
}
} else {
absl::Status status =
collaboration_callbacks_.host_session(name);
if (status.ok()) {
collaboration_state_.active = true;
collaboration_state_.session_id = name;
collaboration_state_.participants.clear();
collaboration_state_.last_synced = absl::Now();
last_collaboration_action_ = absl::Now();
RefreshParticipants();
if (toast_manager_) {
toast_manager_->Show("Hosting collaborative session",
ToastType::kSuccess, 3.5f);
}
MarkHistoryDirty();
} else if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Failed to host: %s", status.message()),
ToastType::kError, 5.0f);
}
}
}
if (!can_host) {
if (ImGui::IsItemHovered()) {
ImGui::SetTooltip("Provide host_session callback to enable hosting");
}
ImGui::EndDisabled();
}
ImGui::InputTextWithHint("##join_code", "Session code",
join_code_buffer_,
IM_ARRAYSIZE(join_code_buffer_));
ImGui::SameLine();
if (!can_join) ImGui::BeginDisabled();
if (ImGui::Button("Join Session")) {
std::string code = join_code_buffer_;
if (code.empty()) {
if (toast_manager_) {
toast_manager_->Show("Enter a session code first",
ToastType::kWarning, 3.0f);
}
} else {
absl::Status status =
collaboration_callbacks_.join_session(code);
if (status.ok()) {
collaboration_state_.active = true;
collaboration_state_.session_id = code;
collaboration_state_.last_synced = absl::Now();
last_collaboration_action_ = absl::Now();
RefreshParticipants();
if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Joined session %s", code.c_str()),
ToastType::kSuccess, 3.5f);
}
MarkHistoryDirty();
} else if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Failed to join: %s", status.message()),
ToastType::kError, 5.0f);
}
}
}
if (!can_join) {
if (ImGui::IsItemHovered()) {
ImGui::SetTooltip("Provide join_session callback to enable joining");
}
ImGui::EndDisabled();
}
if (connected) {
if (!can_leave) ImGui::BeginDisabled();
if (ImGui::Button("Leave Session")) {
absl::Status status = collaboration_callbacks_.leave_session
? collaboration_callbacks_.leave_session()
: absl::OkStatus();
if (status.ok()) {
collaboration_state_ = CollaborationState{};
if (toast_manager_) {
toast_manager_->Show("Left collaborative session",
ToastType::kInfo, 3.0f);
}
MarkHistoryDirty();
} else if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Failed to leave: %s", status.message()),
ToastType::kError, 5.0f);
}
}
if (!can_leave) ImGui::EndDisabled();
}
if (connected) {
ImGui::Separator();
if (ImGui::Button("Refresh Participants")) {
RefreshParticipants();
}
if (collaboration_state_.participants.empty()) {
ImGui::TextDisabled("Awaiting participant list...");
} else {
ImGui::Text("Participants (%zu):",
collaboration_state_.participants.size());
for (const auto& participant : collaboration_state_.participants) {
ImGui::BulletText("%s", participant.c_str());
}
}
} else {
ImGui::TextDisabled("Start or join a session to collaborate in real time.");
}
}
void AgentChatWidget::RenderMultimodalPanel() {
if (!ImGui::CollapsingHeader("Gemini Multimodal (Preview)",
ImGuiTreeNodeFlags_DefaultOpen)) {
return;
}
bool can_capture = static_cast<bool>(multimodal_callbacks_.capture_snapshot);
bool can_send = static_cast<bool>(multimodal_callbacks_.send_to_gemini);
if (!can_capture) ImGui::BeginDisabled();
if (ImGui::Button("Capture Map Snapshot")) {
if (multimodal_callbacks_.capture_snapshot) {
std::filesystem::path captured_path;
absl::Status status =
multimodal_callbacks_.capture_snapshot(&captured_path);
if (status.ok()) {
multimodal_state_.last_capture_path = captured_path;
multimodal_state_.status_message =
absl::StrFormat("Captured %s", captured_path.string());
multimodal_state_.last_updated = absl::Now();
if (toast_manager_) {
toast_manager_->Show("Snapshot captured",
ToastType::kSuccess, 3.0f);
}
MarkHistoryDirty();
} else {
multimodal_state_.status_message = status.message();
multimodal_state_.last_updated = absl::Now();
if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Snapshot failed: %s", status.message()),
ToastType::kError, 5.0f);
}
}
}
}
if (!can_capture) {
if (ImGui::IsItemHovered()) {
ImGui::SetTooltip("Provide capture_snapshot callback to enable");
}
ImGui::EndDisabled();
}
if (multimodal_state_.last_capture_path.has_value()) {
ImGui::TextDisabled("Last capture: %s",
multimodal_state_.last_capture_path->string().c_str());
} else {
ImGui::TextDisabled("No capture yet");
}
ImGui::InputTextMultiline("##gemini_prompt", multimodal_prompt_buffer_,
IM_ARRAYSIZE(multimodal_prompt_buffer_),
ImVec2(-1, 60.0f));
if (!can_send) ImGui::BeginDisabled();
if (ImGui::Button("Send to Gemini")) {
if (!multimodal_state_.last_capture_path.has_value()) {
if (toast_manager_) {
toast_manager_->Show("Capture a snapshot first",
ToastType::kWarning, 3.0f);
}
} else {
std::string prompt = multimodal_prompt_buffer_;
absl::Status status = multimodal_callbacks_.send_to_gemini(
*multimodal_state_.last_capture_path, prompt);
if (status.ok()) {
multimodal_state_.status_message =
"Submitted image to Gemini";
multimodal_state_.last_updated = absl::Now();
if (toast_manager_) {
toast_manager_->Show("Gemini request sent",
ToastType::kSuccess, 3.0f);
}
MarkHistoryDirty();
} else {
multimodal_state_.status_message = status.message();
multimodal_state_.last_updated = absl::Now();
if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Gemini request failed: %s", status.message()),
ToastType::kError, 5.0f);
}
}
}
}
if (!can_send) {
if (ImGui::IsItemHovered()) {
ImGui::SetTooltip("Provide send_to_gemini callback to enable");
}
ImGui::EndDisabled();
}
if (!multimodal_state_.status_message.empty()) {
ImGui::TextDisabled("Status: %s", multimodal_state_.status_message.c_str());
if (multimodal_state_.last_updated != absl::InfinitePast()) {
ImGui::TextDisabled(
"Updated: %s",
absl::FormatTime("%H:%M:%S", multimodal_state_.last_updated,
absl::LocalTimeZone()).c_str());
}
}
}
void AgentChatWidget::RefreshParticipants() {
if (!collaboration_callbacks_.refresh_participants) {
return;
}
auto participants_or = collaboration_callbacks_.refresh_participants();
if (!participants_or.ok()) {
if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Failed to refresh participants: %s",
participants_or.status().message()),
ToastType::kError, 5.0f);
}
return;
}
collaboration_state_.participants = participants_or.value();
collaboration_state_.last_synced = absl::Now();
MarkHistoryDirty();
}
void AgentChatWidget::MarkHistoryDirty() {
history_dirty_ = true;
const absl::Time now = absl::Now();
if (last_persist_time_ == absl::InfinitePast() ||
now - last_persist_time_ > absl::Seconds(2)) {
PersistHistory();
}
}
} // namespace editor
} // namespace yaze

78
src/app/editor/system/agent_chat_widget.h
View File

@@ -2,7 +2,10 @@
#define YAZE_SRC_APP_EDITOR_SYSTEM_AGENT_CHAT_WIDGET_H_
#include <filesystem>
#include <functional>
#include <optional>
#include <string>
#include <vector>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
@@ -26,12 +29,28 @@ class AgentChatWidget {
void SetRomContext(Rom* rom);
void SetToastManager(ToastManager* toast_manager) {
toast_manager_ = toast_manager;
struct CollaborationCallbacks {
std::function<absl::Status(const std::string&)> host_session;
std::function<absl::Status(const std::string&)> join_session;
std::function<absl::Status()> leave_session;
std::function<absl::StatusOr<std::vector<std::string>>()> refresh_participants;
};
struct MultimodalCallbacks {
std::function<absl::Status(std::filesystem::path*)> capture_snapshot;
std::function<absl::Status(const std::filesystem::path&, const std::string&)> send_to_gemini;
};
void SetToastManager(ToastManager* toast_manager);
void SetProposalDrawer(ProposalDrawer* drawer);
void SetCollaborationCallbacks(const CollaborationCallbacks& callbacks) {
collaboration_callbacks_ = callbacks;
}
void SetProposalDrawer(ProposalDrawer* drawer) {
proposal_drawer_ = drawer;
void SetMultimodalCallbacks(const MultimodalCallbacks& callbacks) {
multimodal_callbacks_ = callbacks;
}
bool* active() { return &active_; }
@@ -39,19 +58,36 @@ class AgentChatWidget {
void set_active(bool active) { active_ = active; }
private:
void EnsureHistoryLoaded();
void PersistHistory();
void RenderHistory();
void RenderMessage(const cli::agent::ChatMessage& msg, int index);
void RenderProposalQuickActions(const cli::agent::ChatMessage& msg,
int index);
void RenderInputBox();
void HandleAgentResponse(
const absl::StatusOr<cli::agent::ChatMessage>& response);
int CountKnownProposals() const;
void FocusProposalDrawer(const std::string& proposal_id);
void NotifyProposalCreated(const cli::agent::ChatMessage& msg,
int new_total_proposals);
struct CollaborationState {
bool active = false;
std::string session_id;
std::vector<std::string> participants;
absl::Time last_synced = absl::InfinitePast();
};
struct MultimodalState {
std::optional<std::filesystem::path> last_capture_path;
std::string status_message;
absl::Time last_updated = absl::InfinitePast();
};
void EnsureHistoryLoaded();
void PersistHistory();
void RenderHistory();
void RenderMessage(const cli::agent::ChatMessage& msg, int index);
void RenderProposalQuickActions(const cli::agent::ChatMessage& msg,
int index);
void RenderInputBox();
void HandleAgentResponse(
const absl::StatusOr<cli::agent::ChatMessage>& response);
int CountKnownProposals() const;
void FocusProposalDrawer(const std::string& proposal_id);
void NotifyProposalCreated(const cli::agent::ChatMessage& msg,
int new_total_proposals);
void RenderCollaborationPanel();
void RenderMultimodalPanel();
void RefreshParticipants();
void MarkHistoryDirty();
cli::agent::ConversationalAgentService agent_service_;
char input_buffer_[1024];
@@ -66,6 +102,14 @@ class AgentChatWidget {
ProposalDrawer* proposal_drawer_ = nullptr;
std::string pending_focus_proposal_id_;
absl::Time last_persist_time_ = absl::InfinitePast();
CollaborationState collaboration_state_;
CollaborationCallbacks collaboration_callbacks_;
MultimodalState multimodal_state_;
MultimodalCallbacks multimodal_callbacks_;
char session_name_buffer_[64] = {};
char join_code_buffer_[64] = {};
char multimodal_prompt_buffer_[256] = {};
absl::Time last_collaboration_action_ = absl::InfinitePast();
};
} // namespace editor