feat: Implement auto-capture of screenshots and diagnostics on test failures

- Added a new helper function `CaptureHarnessScreenshot` to encapsulate SDL screenshot logic.
- Updated `ImGuiTestHarnessServiceImpl::Screenshot` to utilize the new screenshot helper.
- Enhanced `TestManager::CaptureFailureContext` to automatically capture screenshots and widget state on test failures.
- Introduced new fields in the `GetTestResultsResponse` proto for screenshot path, size, failure context, and widget state.
- Updated CLI and gRPC client to expose new diagnostic fields in test results.
- Ensured that screenshots are saved in a structured directory under the system's temp directory.
- Improved logging for auto-capture events, including success and failure messages.
This commit is contained in:
scawful
2025-10-02 23:36:09 -04:00
parent c348f7f91f
commit 0447d6f8a1
11 changed files with 509 additions and 419 deletions

View File

@@ -264,6 +264,8 @@ if(YAZE_WITH_GRPC)
target_sources(yaze PRIVATE
${CMAKE_SOURCE_DIR}/src/app/core/service/imgui_test_harness_service.cc
${CMAKE_SOURCE_DIR}/src/app/core/service/imgui_test_harness_service.h
${CMAKE_SOURCE_DIR}/src/app/core/service/screenshot_utils.cc
${CMAKE_SOURCE_DIR}/src/app/core/service/screenshot_utils.h
${CMAKE_SOURCE_DIR}/src/app/core/service/widget_discovery_service.cc
${CMAKE_SOURCE_DIR}/src/app/core/service/widget_discovery_service.h
${CMAKE_SOURCE_DIR}/src/app/core/testing/test_recorder.cc

View File

@@ -24,6 +24,7 @@
#include "absl/time/time.h"
#include "app/core/proto/imgui_test_harness.grpc.pb.h"
#include "app/core/proto/imgui_test_harness.pb.h"
#include "app/core/service/screenshot_utils.h"
#include "app/core/testing/test_script_parser.h"
#include "app/test/test_manager.h"
#include "yaze.h" // For YAZE_VERSION_STRING
@@ -1187,82 +1188,30 @@ absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request,
return finalize(absl::OkStatus());
}
// Helper struct matching imgui_impl_sdlrenderer2.cpp backend data
struct ImGui_ImplSDLRenderer2_Data {
SDL_Renderer* Renderer;
};
absl::Status ImGuiTestHarnessServiceImpl::Screenshot(
const ScreenshotRequest* request, ScreenshotResponse* response) {
// Get the SDL renderer from ImGui backend
ImGuiIO& io = ImGui::GetIO();
auto* backend_data = static_cast<ImGui_ImplSDLRenderer2_Data*>(io.BackendRendererUserData);
if (!backend_data || !backend_data->Renderer) {
if (!response) {
return absl::InvalidArgumentError("response cannot be null");
}
const std::string requested_path =
request ? request->output_path() : std::string();
absl::StatusOr<ScreenshotArtifact> artifact_or =
CaptureHarnessScreenshot(requested_path);
if (!artifact_or.ok()) {
response->set_success(false);
response->set_message("SDL renderer not available");
return absl::FailedPreconditionError("No SDL renderer available");
response->set_message(std::string(artifact_or.status().message()));
return artifact_or.status();
}
SDL_Renderer* renderer = backend_data->Renderer;
// Get renderer output size
int width, height;
if (SDL_GetRendererOutputSize(renderer, &width, &height) != 0) {
response->set_success(false);
response->set_message(absl::StrFormat("Failed to get renderer size: %s", SDL_GetError()));
return absl::InternalError("Failed to get renderer output size");
}
// Create surface to hold screenshot
SDL_Surface* surface = SDL_CreateRGBSurface(0, width, height, 32,
0x00FF0000, 0x0000FF00,
0x000000FF, 0xFF000000);
if (!surface) {
response->set_success(false);
response->set_message(absl::StrFormat("Failed to create surface: %s", SDL_GetError()));
return absl::InternalError("Failed to create SDL surface");
}
// Read pixels from renderer
if (SDL_RenderReadPixels(renderer, nullptr, SDL_PIXELFORMAT_ARGB8888,
surface->pixels, surface->pitch) != 0) {
SDL_FreeSurface(surface);
response->set_success(false);
response->set_message(absl::StrFormat("Failed to read pixels: %s", SDL_GetError()));
return absl::InternalError("Failed to read renderer pixels");
}
// Determine output path
std::string output_path = request->output_path();
if (output_path.empty()) {
// Default: /tmp/yaze_screenshot_<timestamp>.bmp
output_path = absl::StrFormat("/tmp/yaze_screenshot_%lld.bmp",
absl::ToUnixMillis(absl::Now()));
}
// Save to BMP file (SDL built-in, no external deps needed)
if (SDL_SaveBMP(surface, output_path.c_str()) != 0) {
SDL_FreeSurface(surface);
response->set_success(false);
response->set_message(absl::StrFormat("Failed to save BMP: %s", SDL_GetError()));
return absl::InternalError("Failed to save screenshot");
}
// Get file size
std::ifstream file(output_path, std::ios::binary | std::ios::ate);
int64_t file_size = file.tellg();
file.close();
// Clean up and return success
SDL_FreeSurface(surface);
const ScreenshotArtifact& artifact = *artifact_or;
response->set_success(true);
response->set_message(absl::StrFormat("Screenshot saved to %s (%dx%d)",
output_path, width, height));
response->set_file_path(output_path);
response->set_file_size_bytes(file_size);
artifact.file_path, artifact.width,
artifact.height));
response->set_file_path(artifact.file_path);
response->set_file_size_bytes(artifact.file_size_bytes);
return absl::OkStatus();
}

View File

@@ -0,0 +1,108 @@
#include "app/core/service/screenshot_utils.h"
#ifdef YAZE_WITH_GRPC
#include <SDL.h>
#include <filesystem>
#include <string>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "absl/time/clock.h"
#include "imgui.h"
namespace yaze {
namespace test {
namespace {
struct ImGui_ImplSDLRenderer2_Data {
SDL_Renderer* Renderer;
};
std::filesystem::path DefaultScreenshotPath() {
std::filesystem::path base_dir =
std::filesystem::temp_directory_path() / "yaze" / "test-results";
std::error_code ec;
std::filesystem::create_directories(base_dir, ec);
const int64_t timestamp_ms = absl::ToUnixMillis(absl::Now());
return base_dir /
std::filesystem::path(
absl::StrFormat("harness_%lld.bmp", static_cast<long long>(timestamp_ms)));
}
} // namespace
absl::StatusOr<ScreenshotArtifact> CaptureHarnessScreenshot(
const std::string& preferred_path) {
ImGuiIO& io = ImGui::GetIO();
auto* backend_data =
static_cast<ImGui_ImplSDLRenderer2_Data*>(io.BackendRendererUserData);
if (!backend_data || !backend_data->Renderer) {
return absl::FailedPreconditionError("SDL renderer not available");
}
SDL_Renderer* renderer = backend_data->Renderer;
int width = 0;
int height = 0;
if (SDL_GetRendererOutputSize(renderer, &width, &height) != 0) {
return absl::InternalError(
absl::StrFormat("Failed to get renderer size: %s", SDL_GetError()));
}
std::filesystem::path output_path = preferred_path.empty()
? DefaultScreenshotPath()
: std::filesystem::path(preferred_path);
if (output_path.has_parent_path()) {
std::error_code ec;
std::filesystem::create_directories(output_path.parent_path(), ec);
}
SDL_Surface* surface = SDL_CreateRGBSurface(0, width, height, 32, 0x00FF0000,
0x0000FF00, 0x000000FF,
0xFF000000);
if (!surface) {
return absl::InternalError(
absl::StrFormat("Failed to create SDL surface: %s", SDL_GetError()));
}
if (SDL_RenderReadPixels(renderer, nullptr, SDL_PIXELFORMAT_ARGB8888,
surface->pixels, surface->pitch) != 0) {
SDL_FreeSurface(surface);
return absl::InternalError(
absl::StrFormat("Failed to read renderer pixels: %s", SDL_GetError()));
}
if (SDL_SaveBMP(surface, output_path.string().c_str()) != 0) {
SDL_FreeSurface(surface);
return absl::InternalError(
absl::StrFormat("Failed to save BMP: %s", SDL_GetError()));
}
SDL_FreeSurface(surface);
std::error_code ec;
const int64_t file_size =
std::filesystem::file_size(output_path, ec);
if (ec) {
return absl::InternalError(
absl::StrFormat("Failed to stat screenshot %s: %s",
output_path.string(), ec.message()));
}
ScreenshotArtifact artifact;
artifact.file_path = output_path.string();
artifact.width = width;
artifact.height = height;
artifact.file_size_bytes = file_size;
return artifact;
}
} // namespace test
} // namespace yaze
#endif // YAZE_WITH_GRPC

View File

@@ -0,0 +1,31 @@
#ifndef YAZE_APP_CORE_SERVICE_SCREENSHOT_UTILS_H_
#define YAZE_APP_CORE_SERVICE_SCREENSHOT_UTILS_H_
#ifdef YAZE_WITH_GRPC
#include <string>
#include "absl/status/statusor.h"
namespace yaze {
namespace test {
struct ScreenshotArtifact {
std::string file_path;
int width = 0;
int height = 0;
int64_t file_size_bytes = 0;
};
// Captures the current renderer output into a BMP file.
// If preferred_path is empty, an appropriate path under the system temp
// directory is generated automatically. Returns the resolved artifact metadata
// on success.
absl::StatusOr<ScreenshotArtifact> CaptureHarnessScreenshot(
const std::string& preferred_path = "");
} // namespace test
} // namespace yaze
#endif // YAZE_WITH_GRPC
#endif // YAZE_APP_CORE_SERVICE_SCREENSHOT_UTILS_H_

View File

@@ -1,14 +1,17 @@
#include "app/test/test_manager.h"
#include <algorithm>
#include <filesystem>
#include <random>
#include "absl/status/statusor.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_replace.h"
#include "absl/synchronization/mutex.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "app/core/service/screenshot_utils.h"
#include "app/core/widget_state_capture.h"
#include "app/core/features.h"
#include "app/core/platform/file_dialog.h"
@@ -36,6 +39,25 @@ class EditorManager;
namespace yaze {
namespace test {
namespace {
std::string GenerateFailureScreenshotPath(const std::string& test_id) {
std::filesystem::path base_dir =
std::filesystem::temp_directory_path() / "yaze" / "test-results" /
test_id;
std::error_code ec;
std::filesystem::create_directories(base_dir, ec);
const int64_t timestamp_ms = absl::ToUnixMillis(absl::Now());
std::filesystem::path file_path =
base_dir /
std::filesystem::path(absl::StrFormat(
"failure_%lld.bmp", static_cast<long long>(timestamp_ms)));
return file_path.string();
}
} // namespace
// Utility function implementations
const char* TestStatusToString(TestStatus status) {
switch (status) {
@@ -1636,68 +1658,117 @@ void TestManager::TrimHarnessHistoryLocked() {
void TestManager::CaptureFailureContext(const std::string& test_id) {
// IT-08b: Capture failure diagnostics
// Note: This method is called with the harness_history_mutex_ unlocked
// to avoid deadlock when Screenshot RPC calls back into TestManager
// to avoid deadlock when Screenshot helper touches SDL state.
absl::MutexLock lock(&harness_history_mutex_);
auto it = harness_history_.find(test_id);
if (it == harness_history_.end()) {
return;
}
HarnessTestExecution& execution = it->second;
// 1. Capture execution context (frame count, active window, etc.)
// 1. Capture execution context metadata from ImGui.
std::string failure_context;
ImGuiContext* ctx = ImGui::GetCurrentContext();
if (ctx != nullptr) {
#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE
ImGuiWindow* current_window = ctx->CurrentWindow;
ImGuiWindow* nav_window = ctx->NavWindow;
ImGuiWindow* hovered_window = ctx->HoveredWindow;
ImGuiWindow* current_window = ctx->CurrentWindow;
ImGuiWindow* nav_window = ctx->NavWindow;
ImGuiWindow* hovered_window = ctx->HoveredWindow;
const char* current_name =
(current_window && current_window->Name) ? current_window->Name : "none";
const char* nav_name =
(nav_window && nav_window->Name) ? nav_window->Name : "none";
const char* hovered_name =
(hovered_window && hovered_window->Name) ? hovered_window->Name : "none";
const char* current_name =
(current_window && current_window->Name) ? current_window->Name : "none";
const char* nav_name =
(nav_window && nav_window->Name) ? nav_window->Name : "none";
const char* hovered_name = (hovered_window && hovered_window->Name)
? hovered_window->Name
: "none";
ImGuiID active_id = ImGui::GetActiveID();
ImGuiID hovered_id = ImGui::GetHoveredID();
execution.failure_context =
absl::StrFormat(
"frame=%d current_window=%s nav_window=%s hovered_window=%s active_id=0x%08X hovered_id=0x%08X",
ImGui::GetFrameCount(), current_name, nav_name, hovered_name,
active_id, hovered_id);
ImGuiID active_id = ImGui::GetActiveID();
ImGuiID hovered_id = ImGui::GetHoveredID();
failure_context = absl::StrFormat(
"frame=%d current_window=%s nav_window=%s hovered_window=%s "
"active_id=0x%08X hovered_id=0x%08X",
ImGui::GetFrameCount(), current_name, nav_name, hovered_name,
active_id, hovered_id);
#else
execution.failure_context =
absl::StrFormat("frame=%d", ImGui::GetFrameCount());
failure_context =
absl::StrFormat("frame=%d", ImGui::GetFrameCount());
#endif
} else {
execution.failure_context = "ImGui context not available";
failure_context = "ImGui context not available";
}
// 2. Screenshot capture would happen here via gRPC call
// Note: Screenshot RPC implementation is in ImGuiTestHarnessServiceImpl
// The screenshot_path will be set by the RPC handler when it completes
// For now, we just set a placeholder path to indicate where it should be saved
if (execution.screenshot_path.empty()) {
execution.screenshot_path =
absl::StrFormat("/tmp/yaze_test_%s_failure.bmp", test_id);
execution.screenshot_size_bytes = 0;
std::string artifact_path;
{
absl::MutexLock lock(&harness_history_mutex_);
auto it = harness_history_.find(test_id);
if (it == harness_history_.end()) {
return;
}
HarnessTestExecution& execution = it->second;
execution.failure_context = failure_context;
if (execution.screenshot_path.empty()) {
execution.screenshot_path = GenerateFailureScreenshotPath(test_id);
}
artifact_path = execution.screenshot_path;
}
// 3. Widget state capture (IT-08c)
execution.widget_state = core::CaptureWidgetState();
// 2. Capture widget state snapshot (IT-08c) and failure screenshot.
std::string widget_state = core::CaptureWidgetState();
#if defined(YAZE_WITH_GRPC)
absl::StatusOr<ScreenshotArtifact> screenshot_artifact =
CaptureHarnessScreenshot(artifact_path);
#endif
// Keep aggregate cache in sync with the latest execution snapshot.
auto aggregate_it = harness_aggregates_.find(execution.name);
if (aggregate_it != harness_aggregates_.end()) {
aggregate_it->second.latest_execution = execution;
{
absl::MutexLock lock(&harness_history_mutex_);
auto it = harness_history_.find(test_id);
if (it == harness_history_.end()) {
return;
}
HarnessTestExecution& execution = it->second;
execution.failure_context = failure_context;
execution.widget_state = widget_state;
#if defined(YAZE_WITH_GRPC)
if (screenshot_artifact.ok()) {
execution.screenshot_path = screenshot_artifact->file_path;
execution.screenshot_size_bytes = screenshot_artifact->file_size_bytes;
execution.logs.push_back(absl::StrFormat(
"[auto-capture] Failure screenshot saved to %s (%lld bytes)",
execution.screenshot_path,
static_cast<long long>(execution.screenshot_size_bytes)));
} else {
execution.logs.push_back(absl::StrFormat(
"[auto-capture] Screenshot capture failed: %s",
screenshot_artifact.status().message()));
}
#else
execution.logs.push_back(
"[auto-capture] Screenshot capture unavailable (YAZE_WITH_GRPC=OFF)");
#endif
// Keep aggregate cache in sync with the latest execution snapshot.
auto aggregate_it = harness_aggregates_.find(execution.name);
if (aggregate_it != harness_aggregates_.end()) {
aggregate_it->second.latest_execution = execution;
}
}
util::logf("[TestManager] Captured failure context for test %s: %s",
test_id.c_str(), execution.failure_context.c_str());
util::logf("[TestManager] Widget state: %s", execution.widget_state.c_str());
#if defined(YAZE_WITH_GRPC)
if (screenshot_artifact.ok()) {
util::logf("[TestManager] Captured failure context for test %s: %s",
test_id.c_str(), failure_context.c_str());
util::logf("[TestManager] Failure screenshot stored at %s (%lld bytes)",
screenshot_artifact->file_path.c_str(),
static_cast<long long>(screenshot_artifact->file_size_bytes));
} else {
util::logf("[TestManager] Failed to capture screenshot for test %s: %s",
test_id.c_str(),
screenshot_artifact.status().ToString().c_str());
}
#else
util::logf(
"[TestManager] Screenshot capture unavailable (YAZE_WITH_GRPC=OFF) for test %s",
test_id.c_str());
#endif
util::logf("[TestManager] Widget state: %s", widget_state.c_str());
}
} // namespace test

View File

@@ -739,6 +739,30 @@ absl::Status HandleTestResultsCommand(const std::vector<std::string>& arg_vec) {
std::cout << "[],\n";
}
std::cout << " \"screenshot_path\": ";
if (details.screenshot_path.empty()) {
std::cout << "null,\n";
} else {
std::cout << "\"" << JsonEscape(details.screenshot_path) << "\",\n";
}
std::cout << " \"screenshot_size_bytes\": "
<< details.screenshot_size_bytes << ",\n";
std::cout << " \"failure_context\": ";
if (details.failure_context.empty()) {
std::cout << "null,\n";
} else {
std::cout << "\"" << JsonEscape(details.failure_context) << "\",\n";
}
std::cout << " \"widget_state\": ";
if (details.widget_state.empty()) {
std::cout << "null,\n";
} else {
std::cout << "\"" << JsonEscape(details.widget_state) << "\",\n";
}
std::cout << " \"metrics\": ";
if (!details.metrics.empty()) {
std::cout << "{\n";
@@ -807,6 +831,27 @@ absl::Status HandleTestResultsCommand(const std::vector<std::string>& arg_vec) {
std::cout << " " << key << ": " << value << "\n";
}
}
if (details.screenshot_path.empty()) {
std::cout << "screenshot_path: null\n";
} else {
std::cout << "screenshot_path: "
<< YamlQuote(details.screenshot_path) << "\n";
}
std::cout << "screenshot_size_bytes: " << details.screenshot_size_bytes
<< "\n";
if (details.failure_context.empty()) {
std::cout << "failure_context: null\n";
} else {
std::cout << "failure_context: "
<< YamlQuote(details.failure_context) << "\n";
}
if (details.widget_state.empty()) {
std::cout << "widget_state: null\n";
} else {
std::cout << "widget_state: " << YamlQuote(details.widget_state)
<< "\n";
}
}
return absl::OkStatus();

View File

@@ -463,6 +463,11 @@ absl::StatusOr<TestResultDetails> GuiAutomationClient::GetTestResults(
result.metrics.emplace(metric.first, metric.second);
}
result.screenshot_path = response.screenshot_path();
result.screenshot_size_bytes = response.screenshot_size_bytes();
result.failure_context = response.failure_context();
result.widget_state = response.widget_state();
return result;
#else
return absl::UnimplementedError("gRPC not available");

View File

@@ -119,6 +119,10 @@ struct TestResultDetails {
std::vector<AssertionOutcome> assertions;
std::vector<std::string> logs;
std::map<std::string, int> metrics;
std::string screenshot_path;
int64_t screenshot_size_bytes = 0;
std::string failure_context;
std::string widget_state;
};
enum class WidgetTypeFilter {