feat: Implement screenshot capture functionality for specific regions and active windows in screenshot_utils

- Added CaptureHarnessScreenshotRegion function to capture a specified region of the renderer output.
- Introduced CaptureActiveWindow and CaptureWindowByName functions to capture screenshots of the currently active ImGui window or a specific window by name.
- Updated screenshot_utils.h to include new CaptureRegion struct and corresponding function declarations.
- Enhanced EditorManager to utilize the new capture modes based on user selection in the AgentChatWidget.
This commit is contained in:
scawful
2025-10-04 17:08:36 -04:00
parent 5f2c72bfc7
commit f182833afe
6 changed files with 282 additions and 12 deletions

View File

@@ -12,6 +12,7 @@
#include "absl/strings/str_format.h"
#include "absl/time/clock.h"
#include "imgui.h"
#include "imgui_internal.h"
namespace yaze {
namespace test {
@@ -102,6 +103,144 @@ absl::StatusOr<ScreenshotArtifact> CaptureHarnessScreenshot(
return artifact;
}
absl::StatusOr<ScreenshotArtifact> CaptureHarnessScreenshotRegion(
const std::optional<CaptureRegion>& region,
const std::string& preferred_path) {
ImGuiIO& io = ImGui::GetIO();
auto* backend_data =
static_cast<ImGui_ImplSDLRenderer2_Data*>(io.BackendRendererUserData);
if (!backend_data || !backend_data->Renderer) {
return absl::FailedPreconditionError("SDL renderer not available");
}
SDL_Renderer* renderer = backend_data->Renderer;
// Get full renderer size
int full_width = 0;
int full_height = 0;
if (SDL_GetRendererOutputSize(renderer, &full_width, &full_height) != 0) {
return absl::InternalError(
absl::StrFormat("Failed to get renderer size: %s", SDL_GetError()));
}
// Determine capture region
int capture_x = 0;
int capture_y = 0;
int capture_width = full_width;
int capture_height = full_height;
if (region.has_value()) {
capture_x = region->x;
capture_y = region->y;
capture_width = region->width;
capture_height = region->height;
// Clamp to renderer bounds
if (capture_x < 0) capture_x = 0;
if (capture_y < 0) capture_y = 0;
if (capture_x + capture_width > full_width) {
capture_width = full_width - capture_x;
}
if (capture_y + capture_height > full_height) {
capture_height = full_height - capture_y;
}
if (capture_width <= 0 || capture_height <= 0) {
return absl::InvalidArgumentError("Invalid capture region");
}
}
std::filesystem::path output_path = preferred_path.empty()
? DefaultScreenshotPath()
: std::filesystem::path(preferred_path);
if (output_path.has_parent_path()) {
std::error_code ec;
std::filesystem::create_directories(output_path.parent_path(), ec);
}
// Create surface for the capture region
SDL_Surface* surface = SDL_CreateRGBSurface(0, capture_width, capture_height,
32, 0x00FF0000, 0x0000FF00,
0x000000FF, 0xFF000000);
if (!surface) {
return absl::InternalError(
absl::StrFormat("Failed to create SDL surface: %s", SDL_GetError()));
}
// Read pixels from the specified region
SDL_Rect region_rect = {capture_x, capture_y, capture_width, capture_height};
if (SDL_RenderReadPixels(renderer, &region_rect, SDL_PIXELFORMAT_ARGB8888,
surface->pixels, surface->pitch) != 0) {
SDL_FreeSurface(surface);
return absl::InternalError(
absl::StrFormat("Failed to read renderer pixels: %s", SDL_GetError()));
}
if (SDL_SaveBMP(surface, output_path.string().c_str()) != 0) {
SDL_FreeSurface(surface);
return absl::InternalError(
absl::StrFormat("Failed to save BMP: %s", SDL_GetError()));
}
SDL_FreeSurface(surface);
std::error_code ec;
const int64_t file_size = std::filesystem::file_size(output_path, ec);
if (ec) {
return absl::InternalError(
absl::StrFormat("Failed to stat screenshot %s: %s",
output_path.string(), ec.message()));
}
ScreenshotArtifact artifact;
artifact.file_path = output_path.string();
artifact.width = capture_width;
artifact.height = capture_height;
artifact.file_size_bytes = file_size;
return artifact;
}
absl::StatusOr<ScreenshotArtifact> CaptureActiveWindow(
const std::string& preferred_path) {
ImGuiContext* ctx = ImGui::GetCurrentContext();
if (!ctx || !ctx->NavWindow) {
return absl::FailedPreconditionError("No active ImGui window");
}
ImGuiWindow* window = ctx->NavWindow;
CaptureRegion region;
region.x = static_cast<int>(window->Pos.x);
region.y = static_cast<int>(window->Pos.y);
region.width = static_cast<int>(window->Size.x);
region.height = static_cast<int>(window->Size.y);
return CaptureHarnessScreenshotRegion(region, preferred_path);
}
absl::StatusOr<ScreenshotArtifact> CaptureWindowByName(
const std::string& window_name,
const std::string& preferred_path) {
ImGuiContext* ctx = ImGui::GetCurrentContext();
if (!ctx) {
return absl::FailedPreconditionError("No ImGui context");
}
ImGuiWindow* window = ImGui::FindWindowByName(window_name.c_str());
if (!window) {
return absl::NotFoundError(
absl::StrFormat("Window '%s' not found", window_name));
}
CaptureRegion region;
region.x = static_cast<int>(window->Pos.x);
region.y = static_cast<int>(window->Pos.y);
region.width = static_cast<int>(window->Size.x);
region.height = static_cast<int>(window->Size.y);
return CaptureHarnessScreenshotRegion(region, preferred_path);
}
} // namespace test
} // namespace yaze

View File

@@ -3,6 +3,7 @@
#ifdef YAZE_WITH_GRPC
#include <optional>
#include <string>
#include "absl/status/statusor.h"
@@ -17,6 +18,13 @@ struct ScreenshotArtifact {
int64_t file_size_bytes = 0;
};
struct CaptureRegion {
int x = 0;
int y = 0;
int width = 0;
int height = 0;
};
// Captures the current renderer output into a BMP file.
// If preferred_path is empty, an appropriate path under the system temp
// directory is generated automatically. Returns the resolved artifact metadata
@@ -24,6 +32,21 @@ struct ScreenshotArtifact {
absl::StatusOr<ScreenshotArtifact> CaptureHarnessScreenshot(
const std::string& preferred_path = "");
// Captures a specific region of the renderer output.
// If region is nullopt, captures the full renderer (same as CaptureHarnessScreenshot).
absl::StatusOr<ScreenshotArtifact> CaptureHarnessScreenshotRegion(
const std::optional<CaptureRegion>& region,
const std::string& preferred_path = "");
// Captures the currently active ImGui window.
absl::StatusOr<ScreenshotArtifact> CaptureActiveWindow(
const std::string& preferred_path = "");
// Captures a specific ImGui window by name.
absl::StatusOr<ScreenshotArtifact> CaptureWindowByName(
const std::string& window_name,
const std::string& preferred_path = "");
} // namespace test
} // namespace yaze

View File

@@ -287,8 +287,43 @@ void EditorManager::Initialize(const std::string& filename) {
// Set up multimodal (vision) callbacks for Gemini
AgentChatWidget::MultimodalCallbacks multimodal_callbacks;
multimodal_callbacks.capture_snapshot =
[](std::filesystem::path* output_path) -> absl::Status {
auto result = yaze::test::CaptureHarnessScreenshot("");
[this](std::filesystem::path* output_path) -> absl::Status {
using CaptureMode = AgentChatWidget::CaptureMode;
absl::StatusOr<yaze::test::ScreenshotArtifact> result;
// Capture based on selected mode
switch (agent_chat_widget_.capture_mode()) {
case CaptureMode::kFullWindow:
result = yaze::test::CaptureHarnessScreenshot("");
break;
case CaptureMode::kActiveEditor:
result = yaze::test::CaptureActiveWindow("");
if (!result.ok()) {
// Fallback to full window if no active window
result = yaze::test::CaptureHarnessScreenshot("");
}
break;
case CaptureMode::kSpecificWindow: {
const char* window_name = agent_chat_widget_.specific_window_name();
if (window_name && std::strlen(window_name) > 0) {
result = yaze::test::CaptureWindowByName(window_name, "");
if (!result.ok()) {
// Fallback to active window if specific window not found
result = yaze::test::CaptureActiveWindow("");
}
} else {
result = yaze::test::CaptureActiveWindow("");
}
if (!result.ok()) {
result = yaze::test::CaptureHarnessScreenshot("");
}
break;
}
}
if (!result.ok()) {
return result.status();
}

View File

@@ -27,8 +27,8 @@ absl::Time ParseTimestamp(const Json& value) {
return absl::Now();
}
absl::Time parsed;
if (absl::ParseTime(absl::RFC3339_full, value.get<std::string>(),
absl::UTCTimeZone(), &parsed)) {
if (absl::ParseTime(absl::RFC3339_full, value.get<std::string>(), &parsed,
nullptr)) {
return parsed;
}
return absl::Now();

View File

@@ -484,6 +484,9 @@ void AgentChatWidget::Draw() {
}
EnsureHistoryLoaded();
// Poll for new messages in collaborative sessions
PollSharedHistory();
ImGui::Begin(title_.c_str(), &active_);
RenderHistory();
@@ -668,8 +671,31 @@ void AgentChatWidget::RenderMultimodalPanel() {
bool can_capture = static_cast<bool>(multimodal_callbacks_.capture_snapshot);
bool can_send = static_cast<bool>(multimodal_callbacks_.send_to_gemini);
// Capture mode selection
ImGui::Text("Capture Mode:");
ImGui::RadioButton("Full Window",
reinterpret_cast<int*>(&multimodal_state_.capture_mode),
static_cast<int>(CaptureMode::kFullWindow));
ImGui::SameLine();
ImGui::RadioButton("Active Editor",
reinterpret_cast<int*>(&multimodal_state_.capture_mode),
static_cast<int>(CaptureMode::kActiveEditor));
ImGui::SameLine();
ImGui::RadioButton("Specific Window",
reinterpret_cast<int*>(&multimodal_state_.capture_mode),
static_cast<int>(CaptureMode::kSpecificWindow));
// If specific window mode, show input for window name
if (multimodal_state_.capture_mode == CaptureMode::kSpecificWindow) {
ImGui::InputText("Window Name", multimodal_state_.specific_window_buffer,
IM_ARRAYSIZE(multimodal_state_.specific_window_buffer));
ImGui::TextDisabled("Examples: Overworld Editor, Dungeon Editor, Sprite Editor");
}
ImGui::Separator();
if (!can_capture) ImGui::BeginDisabled();
if (ImGui::Button("Capture Map Snapshot")) {
if (ImGui::Button("Capture Snapshot")) {
if (multimodal_callbacks_.capture_snapshot) {
std::filesystem::path captured_path;
absl::Status status =
@@ -822,6 +848,10 @@ void AgentChatWidget::SwitchToSharedHistory(const std::string& session_id) {
// Load shared history
EnsureHistoryLoaded();
// Initialize polling state
last_known_history_size_ = agent_service_.GetHistory().size();
last_shared_history_poll_ = absl::Now();
if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("Switched to shared chat history for session %s",
@@ -849,5 +879,44 @@ void AgentChatWidget::SwitchToLocalHistory() {
}
}
void AgentChatWidget::PollSharedHistory() {
if (!collaboration_state_.active) {
return; // Not in a collaborative session
}
const absl::Time now = absl::Now();
// Poll every 2 seconds
if (now - last_shared_history_poll_ < absl::Seconds(2)) {
return;
}
last_shared_history_poll_ = now;
// Check if the shared history file has been updated
auto result = AgentChatHistoryCodec::Load(history_path_);
if (!result.ok()) {
return; // File might not exist yet or be temporarily locked
}
const size_t new_size = result->history.size();
// If history has grown, reload it
if (new_size > last_known_history_size_) {
const size_t new_messages = new_size - last_known_history_size_;
agent_service_.ReplaceHistory(std::move(result->history));
last_history_size_ = new_size;
last_known_history_size_ = new_size;
if (toast_manager_) {
toast_manager_->Show(
absl::StrFormat("📬 %zu new message%s from collaborators",
new_messages, new_messages == 1 ? "" : "s"),
ToastType::kInfo, 3.0f);
}
}
}
} // namespace editor
} // namespace yaze

View File

@@ -62,11 +62,6 @@ class AgentChatWidget {
bool* active() { return &active_; }
bool is_active() const { return active_; }
void set_active(bool active) { active_ = active; }
CaptureMode capture_mode() const { return multimodal_state_.capture_mode; }
const char* specific_window_name() const {
return multimodal_state_.specific_window_buffer;
}
public:
struct CollaborationState {
@@ -91,10 +86,19 @@ public:
char specific_window_buffer[128] = {};
};
void EnsureHistoryLoaded();
void PersistHistory();
// Accessors for capture settings
CaptureMode capture_mode() const { return multimodal_state_.capture_mode; }
const char* specific_window_name() const {
return multimodal_state_.specific_window_buffer;
}
// Collaboration history management (public so EditorManager can call them)
void SwitchToSharedHistory(const std::string& session_id);
void SwitchToLocalHistory();
private:
void EnsureHistoryLoaded();
void PersistHistory();
void RenderHistory();
void RenderMessage(const cli::agent::ChatMessage& msg, int index);
void RenderProposalQuickActions(const cli::agent::ChatMessage& msg,