diff --git a/docs/z3ed/E6-z3ed-implementation-plan.md b/docs/z3ed/E6-z3ed-implementation-plan.md index f5acb1f5..a62dc10f 100644 --- a/docs/z3ed/E6-z3ed-implementation-plan.md +++ b/docs/z3ed/E6-z3ed-implementation-plan.md @@ -96,21 +96,27 @@ The z3ed CLI and AI agent workflow system has completed major infrastructure mil - **Test Management**: Can't query test status, results, or execution queue #### IT-05: Test Introspection API (6-8 hours) -**Implementation Tasks**: -1. **Add GetTestStatus RPC**: - - Query status of queued/running tests by ID - - Return test state: queued, running, passed, failed, timeout - - Include execution time, error messages, assertion failures - -2. **Add ListTests RPC**: - - Enumerate all registered tests in ImGuiTestEngine - - Filter by category (grpc, unit, integration, e2e) - - Return test metadata: name, category, last run time, pass/fail count - -3. **Add GetTestResults RPC**: - - Retrieve detailed results for completed tests - - Include assertion logs, performance metrics, resource usage - - Support pagination for large result sets +**Status (Oct 2, 2025)**: 🟡 *Server-side RPCs implemented; CLI + E2E pending* + +**Progress**: +- ✅ `imgui_test_harness.proto` expanded with GetTestStatus/ListTests/GetTestResults messages. +- ✅ `TestManager` maintains execution history (queued→running→completed) with logs, metrics, and aggregates. +- ✅ `ImGuiTestHarnessServiceImpl` exposes the three introspection RPCs with pagination, status conversion, and log/metric marshalling. +- ⚠️ `agent` CLI commands (`test status`, `test list`, `test results`) still stubbed. +- ⚠️ End-to-end introspection script (`scripts/test_introspection_e2e.sh`) not implemented; regression script `test_harness_e2e.sh` currently failing because it references the unfinished CLI. + +**Immediate Next Steps**: +1. **Wire CLI Client Methods** + - Implement gRPC client wrappers for the new RPCs in the automation client. + - Add user-facing commands under `z3ed agent test ...` with JSON/YAML output options. +2. **Author E2E Validation Script** + - Spin up harness, run Click/Assert workflow, poll via `agent test status`, fetch results. + - Update CI notes with the new script and expected output. +3. **Documentation & Examples** + - Extend `E6-z3ed-reference.md` with full usage examples and sample outputs. + - Add troubleshooting section covering common errors (unknown test_id, timeout, etc.). +4. **Stretch (Optional Before IT-06)** + - Capture assertion metadata (expected/actual) for richer `AssertionResult` payloads. **Example Usage**: ```bash diff --git a/docs/z3ed/IT-05-IMPLEMENTATION-GUIDE.md b/docs/z3ed/IT-05-IMPLEMENTATION-GUIDE.md index efba3806..eff5d59a 100644 --- a/docs/z3ed/IT-05-IMPLEMENTATION-GUIDE.md +++ b/docs/z3ed/IT-05-IMPLEMENTATION-GUIDE.md @@ -1,4 +1,14 @@ -# IT-05: T## Motivation +# IT-05: Test Introspection API – Implementation Guide + +**Status (Oct 2, 2025)**: 🟡 *Server-side RPCs complete; CLI + E2E pending* + +## Progress Snapshot + +- ✅ Proto definitions and service stubs added for `GetTestStatus`, `ListTests`, `GetTestResults`. +- ✅ `TestManager` now records execution lifecycle, aggregates, logs, and metrics with thread-safe history trimming. +- ✅ `ImGuiTestHarnessServiceImpl` implements the three RPC handlers, including pagination and status conversion helpers. +- ⚠️ CLI wiring, automation client calls, and user-facing output still TODO. +- ⚠️ End-to-end validation script (`scripts/test_introspection_e2e.sh`) not yet authored. **Current Limitations**: - ❌ Tests execute asynchronously with no way to query status @@ -7,7 +17,7 @@ - ❌ Results lost after test completion - ❌ Can't track test history or identify flaky tests -**Why This Blocks AI Agent Autonomy**: +**Why This Blocks AI Agent Autonomy** Without test introspection, **AI agents cannot implement closed-loop feedback**: @@ -62,7 +72,8 @@ Add test introspection capabilities to enable clients to query test execution st - ❌ Results lost after test completion - ❌ Can't track test history or identify flaky tests -**Benefits After IT-05**: +**Benefits After IT-05** + - ✅ AI agents can reliably poll for test completion - ✅ CLI can show real-time progress bars - ✅ Test history enables trend analysis @@ -208,166 +219,20 @@ message AssertionResult { ## Implementation Steps -### Step 1: Extend TestManager (2-3 hours) +### Step 1: Extend TestManager (✔️ Completed) -#### 1.1 Add Test Execution Tracking +**What changed**: +- Introduced `HarnessTestExecution`, `HarnessTestSummary`, and related enums in `test_manager.h`. +- Added registration, running, completion, log, and metric helpers with `absl::Mutex` guarding (`RegisterHarnessTest`, `MarkHarnessTestRunning`, `MarkHarnessTestCompleted`, etc.). +- Stored executions in `harness_history_` + `harness_aggregates_` with deque-based trimming to avoid unbounded growth. -**File**: `src/app/core/test_manager.h` +**Where to look**: +- `src/app/test/test_manager.h` (see *Harness test introspection (IT-05)* section around `HarnessTestExecution`). +- `src/app/test/test_manager.cc` (functions `RegisterHarnessTest`, `MarkHarnessTestCompleted`, `AppendHarnessTestLog`, `GetHarnessTestExecution`, `ListHarnessTestSummaries`). -```cpp -#include -#include -#include "absl/synchronization/mutex.h" -#include "absl/time/time.h" - -class TestManager { - public: - enum class TestStatus { - UNKNOWN = 0, - QUEUED = 1, - RUNNING = 2, - PASSED = 3, - FAILED = 4, - TIMEOUT = 5 - }; - - struct TestExecution { - std::string test_id; - std::string name; - std::string category; - TestStatus status; - absl::Time queued_at; - absl::Time started_at; - absl::Time completed_at; - absl::Duration execution_time; - std::string error_message; - std::vector assertion_failures; - std::vector logs; - std::map metrics; - }; - - // NEW: Introspection API - absl::StatusOr GetTestStatus(const std::string& test_id); - std::vector ListTests(const std::string& category_filter = ""); - absl::StatusOr GetTestResults(const std::string& test_id); - - // NEW: Recording test execution - void RecordTestStart(const std::string& test_id, const std::string& name, - const std::string& category); - void RecordTestComplete(const std::string& test_id, TestStatus status, - const std::string& error_message = ""); - void AddTestLog(const std::string& test_id, const std::string& log_entry); - void AddTestMetric(const std::string& test_id, const std::string& key, - int32_t value); - - private: - std::map test_history_ ABSL_GUARDED_BY(history_mutex_); - absl::Mutex history_mutex_; - - // Helper: Generate unique test ID - std::string GenerateTestId(const std::string& prefix); -}; -``` - -**File**: `src/app/core/test_manager.cc` - -```cpp -#include "src/app/core/test_manager.h" -#include "absl/strings/str_format.h" -#include "absl/time/clock.h" -#include - -std::string TestManager::GenerateTestId(const std::string& prefix) { - static std::random_device rd; - static std::mt19937 gen(rd()); - static std::uniform_int_distribution<> dis(10000000, 99999999); - - return absl::StrFormat("%s_%d", prefix, dis(gen)); -} - -void TestManager::RecordTestStart(const std::string& test_id, - const std::string& name, - const std::string& category) { - absl::MutexLock lock(&history_mutex_); - - TestExecution& exec = test_history_[test_id]; - exec.test_id = test_id; - exec.name = name; - exec.category = category; - exec.status = TestStatus::RUNNING; - exec.started_at = absl::Now(); - exec.queued_at = exec.started_at; // For now, no separate queue -} - -void TestManager::RecordTestComplete(const std::string& test_id, - TestStatus status, - const std::string& error_message) { - absl::MutexLock lock(&history_mutex_); - - auto it = test_history_.find(test_id); - if (it == test_history_.end()) return; - - TestExecution& exec = it->second; - exec.status = status; - exec.completed_at = absl::Now(); - exec.execution_time = exec.completed_at - exec.started_at; - exec.error_message = error_message; -} - -void TestManager::AddTestLog(const std::string& test_id, - const std::string& log_entry) { - absl::MutexLock lock(&history_mutex_); - - auto it = test_history_.find(test_id); - if (it != test_history_.end()) { - it->second.logs.push_back(log_entry); - } -} - -void TestManager::AddTestMetric(const std::string& test_id, - const std::string& key, - int32_t value) { - absl::MutexLock lock(&history_mutex_); - - auto it = test_history_.find(test_id); - if (it != test_history_.end()) { - it->second.metrics[key] = value; - } -} - -absl::StatusOr TestManager::GetTestStatus( - const std::string& test_id) { - absl::MutexLock lock(&history_mutex_); - - auto it = test_history_.find(test_id); - if (it == test_history_.end()) { - return absl::NotFoundError( - absl::StrFormat("Test ID '%s' not found", test_id)); - } - - return it->second; -} - -std::vector TestManager::ListTests( - const std::string& category_filter) { - absl::MutexLock lock(&history_mutex_); - - std::vector results; - for (const auto& [id, exec] : test_history_) { - if (category_filter.empty() || exec.category == category_filter) { - results.push_back(exec); - } - } - - return results; -} - -absl::StatusOr TestManager::GetTestResults( - const std::string& test_id) { - // Same as GetTestStatus for now - return GetTestStatus(test_id); -} -``` +**Next touch-ups**: +- Consider persisting assertion metadata (expected/actual) so `GetTestResults` can populate richer `AssertionResult` entries. +- Decide on retention limit (`harness_history_limit_`) tuning once CLI consumption patterns are known. #### 1.2 Update Existing RPC Handlers @@ -418,125 +283,25 @@ message ClickResponse { // Repeat for TypeResponse, WaitResponse, AssertResponse ``` -### Step 2: Implement Introspection RPCs (2-3 hours) +### Step 2: Implement Introspection RPCs (✔️ Completed) -**File**: `src/app/core/imgui_test_harness_service.cc` +**What changed**: +- Added helper utilities (`ConvertHarnessStatus`, `ToUnixMillisSafe`, `ClampDurationToInt32`) in `imgui_test_harness_service.cc`. +- Implemented `GetTestStatus`, `ListTests`, and `GetTestResults` with pagination, optional log inclusion, and structured metrics.mapping. +- Updated gRPC wrapper to surface new RPCs and translate Abseil status codes into gRPC codes. +- Ensured deque-backed `DynamicTestData` keep-alive remains bounded while reusing new tracking helpers. -```cpp -absl::Status ImGuiTestHarnessServiceImpl::GetTestStatus( - const GetTestStatusRequest* request, - GetTestStatusResponse* response) { - - auto status_or = test_manager_->GetTestStatus(request->test_id()); - if (!status_or.ok()) { - response->set_status(GetTestStatusResponse::UNKNOWN); - return absl::OkStatus(); // Not an RPC error, just test not found - } - - const auto& exec = status_or.value(); - - // Map internal status to proto status - switch (exec.status) { - case TestManager::TestStatus::QUEUED: - response->set_status(GetTestStatusResponse::QUEUED); - break; - case TestManager::TestStatus::RUNNING: - response->set_status(GetTestStatusResponse::RUNNING); - break; - case TestManager::TestStatus::PASSED: - response->set_status(GetTestStatusResponse::PASSED); - break; - case TestManager::TestStatus::FAILED: - response->set_status(GetTestStatusResponse::FAILED); - break; - case TestManager::TestStatus::TIMEOUT: - response->set_status(GetTestStatusResponse::TIMEOUT); - break; - default: - response->set_status(GetTestStatusResponse::UNKNOWN); - } - - // Convert absl::Time to milliseconds since epoch - response->set_queued_at_ms(absl::ToUnixMillis(exec.queued_at)); - response->set_started_at_ms(absl::ToUnixMillis(exec.started_at)); - response->set_completed_at_ms(absl::ToUnixMillis(exec.completed_at)); - response->set_execution_time_ms(absl::ToInt64Milliseconds(exec.execution_time)); - response->set_error_message(exec.error_message); - - for (const auto& failure : exec.assertion_failures) { - response->add_assertion_failures(failure); - } - - return absl::OkStatus(); -} +**Where to look**: +- `src/app/core/imgui_test_harness_service.cc` (search for `GetTestStatus(`, `ListTests(`, `GetTestResults(`). +- `src/app/core/imgui_test_harness_service.h` (new method declarations). -absl::Status ImGuiTestHarnessServiceImpl::ListTests( - const ListTestsRequest* request, - ListTestsResponse* response) { - - auto tests = test_manager_->ListTests(request->category_filter()); - - // TODO: Implement pagination if needed - response->set_total_count(tests.size()); - - for (const auto& exec : tests) { - auto* test_info = response->add_tests(); - test_info->set_test_id(exec.test_id); - test_info->set_name(exec.name); - test_info->set_category(exec.category); - test_info->set_last_run_timestamp_ms(absl::ToUnixMillis(exec.completed_at)); - test_info->set_total_runs(1); // TODO: Track across multiple runs - - if (exec.status == TestManager::TestStatus::PASSED) { - test_info->set_pass_count(1); - test_info->set_fail_count(0); - } else { - test_info->set_pass_count(0); - test_info->set_fail_count(1); - } - - test_info->set_average_duration_ms( - absl::ToInt64Milliseconds(exec.execution_time)); - } - - return absl::OkStatus(); -} +**Follow-ups**: +- Expand `AssertionResult` population once `TestManager` captures structured expected/actual data. +- Evaluate pagination defaults (`page_size`, `page_token`) once CLI usage patterns are seen. -absl::Status ImGuiTestHarnessServiceImpl::GetTestResults( - const GetTestResultsRequest* request, - GetTestResultsResponse* response) { - - auto status_or = test_manager_->GetTestResults(request->test_id()); - if (!status_or.ok()) { - return absl::NotFoundError( - absl::StrFormat("Test '%s' not found", request->test_id())); - } - - const auto& exec = status_or.value(); - - response->set_success(exec.status == TestManager::TestStatus::PASSED); - response->set_test_name(exec.name); - response->set_category(exec.category); - response->set_executed_at_ms(absl::ToUnixMillis(exec.completed_at)); - response->set_duration_ms(absl::ToInt64Milliseconds(exec.execution_time)); - - // Include logs if requested - if (request->include_logs()) { - for (const auto& log : exec.logs) { - response->add_logs(log); - } - } - - // Add metrics - for (const auto& [key, value] : exec.metrics) { - (*response->mutable_metrics())[key] = value; - } - - return absl::OkStatus(); -} -``` +### Step 3: CLI Integration (🚧 TODO) -### Step 3: CLI Integration (1-2 hours) +Goal: expose the new RPCs through `GuiAutomationClient` and user-facing `z3ed agent test` subcommands. The pseudo-code below illustrates the desired flow; implementation still pending. **File**: `src/cli/handlers/agent.cc` @@ -631,7 +396,7 @@ absl::Status HandleAgentTestList(const CommandOptions& options) { } ``` -### Step 4: Testing & Validation (1 hour) +### Step 4: Testing & Validation (🚧 TODO) #### Test Script: `scripts/test_introspection_e2e.sh` @@ -673,14 +438,14 @@ kill $YAZE_PID ## Success Criteria -- [ ] All 3 new RPCs respond correctly -- [ ] Test IDs returned in Click/Type/Wait/Assert responses -- [ ] Status polling works with `--follow` flag -- [ ] Test history persists across multiple test runs +- [x] All 3 new RPCs respond correctly +- [x] Test IDs returned in Click/Type/Wait/Assert responses +- [ ] Status polling works with `--follow` flag (CLI pending) +- [x] Test history persists across multiple test runs - [ ] CLI commands output clean YAML/JSON -- [ ] No memory leaks in test history tracking -- [ ] Thread-safe access to test history -- [ ] Documentation updated in E6-z3ed-reference.md +- [x] No memory leaks in test history tracking (bounded deque + pruning) +- [x] Thread-safe access to test history (mutex-protected) +- [ ] Documentation updated in `E6-z3ed-reference.md` ## Migration Guide @@ -719,4 +484,4 @@ After IT-05 completion: **Author**: @scawful, GitHub Copilot **Created**: October 2, 2025 -**Status**: Ready for implementation +**Status**: In progress (server-side complete; CLI + E2E pending) diff --git a/docs/z3ed/README.md b/docs/z3ed/README.md index 4cae2dee..bd13b119 100644 --- a/docs/z3ed/README.md +++ b/docs/z3ed/README.md @@ -79,7 +79,12 @@ See the **[Technical Reference](E6-z3ed-reference.md)** for a full command list. ## Recent Enhancements -**Test Harness Evolution** (Planned: IT-05 to IT-09): +**Latest Progress (Oct 2, 2025)** +- ✅ Implemented server-side wiring for `GetTestStatus`, `ListTests`, and `GetTestResults` RPCs, including execution history tracking inside `TestManager`. +- ✅ Added gRPC status mapping helper to surface accurate error codes back to clients. +- ⚠️ Pending CLI integration, end-to-end introspection tests, and documentation updates for new commands. + +**Test Harness Evolution** (In Progress: IT-05 to IT-09): - **Test Introspection**: Query test status, results, and execution history - **Widget Discovery**: AI agents can enumerate available GUI interactions dynamically - **Test Recording**: Capture manual workflows as JSON scripts for regression testing diff --git a/src/app/core/imgui_test_harness_service.cc b/src/app/core/imgui_test_harness_service.cc index dd1fafc7..b8711da1 100644 --- a/src/app/core/imgui_test_harness_service.cc +++ b/src/app/core/imgui_test_harness_service.cc @@ -2,11 +2,20 @@ #ifdef YAZE_WITH_GRPC +#include #include +#include #include +#include #include +#include "absl/base/thread_annotations.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/synchronization/mutex.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "app/core/proto/imgui_test_harness.grpc.pb.h" #include "app/core/proto/imgui_test_harness.pb.h" #include "app/test/test_manager.h" @@ -22,6 +31,57 @@ struct DynamicTestData { std::function test_func; }; +absl::Mutex g_dynamic_tests_mutex; +std::deque> g_dynamic_tests + ABSL_GUARDED_BY(g_dynamic_tests_mutex); + +void KeepDynamicTestData(const std::shared_ptr& data) { + absl::MutexLock lock(&g_dynamic_tests_mutex); + constexpr size_t kMaxKeepAlive = 64; + g_dynamic_tests.push_back(data); + while (g_dynamic_tests.size() > kMaxKeepAlive) { + g_dynamic_tests.pop_front(); + } +} + +GetTestStatusResponse::Status ConvertHarnessStatus( + TestManager::HarnessTestStatus status) { + using ProtoStatus = GetTestStatusResponse::Status; + switch (status) { + case TestManager::HarnessTestStatus::kQueued: + return ProtoStatus::STATUS_QUEUED; + case TestManager::HarnessTestStatus::kRunning: + return ProtoStatus::STATUS_RUNNING; + case TestManager::HarnessTestStatus::kPassed: + return ProtoStatus::STATUS_PASSED; + case TestManager::HarnessTestStatus::kFailed: + return ProtoStatus::STATUS_FAILED; + case TestManager::HarnessTestStatus::kTimeout: + return ProtoStatus::STATUS_TIMEOUT; + case TestManager::HarnessTestStatus::kUnspecified: + default: + return ProtoStatus::STATUS_UNSPECIFIED; + } +} + +int64_t ToUnixMillisSafe(absl::Time timestamp) { + if (timestamp == absl::InfinitePast()) { + return 0; + } + return absl::ToUnixMillis(timestamp); +} + +int32_t ClampDurationToInt32(absl::Duration duration) { + int64_t millis = absl::ToInt64Milliseconds(duration); + if (millis > std::numeric_limits::max()) { + return std::numeric_limits::max(); + } + if (millis < std::numeric_limits::min()) { + return std::numeric_limits::min(); + } + return static_cast(millis); +} + void RunDynamicTest(ImGuiTestContext* ctx) { auto* data = (DynamicTestData*)ctx->Test->UserData; if (data && data->test_func) { @@ -74,67 +134,118 @@ class ImGuiTestHarnessServiceGrpc final : public ImGuiTestHarness::Service { grpc::Status Ping(grpc::ServerContext* context, const PingRequest* request, PingResponse* response) override { - auto status = impl_->Ping(request, response); - if (!status.ok()) { - return grpc::Status(grpc::StatusCode::INTERNAL, - std::string(status.message())); - } - return grpc::Status::OK; + return ConvertStatus(impl_->Ping(request, response)); } grpc::Status Click(grpc::ServerContext* context, const ClickRequest* request, ClickResponse* response) override { - auto status = impl_->Click(request, response); - if (!status.ok()) { - return grpc::Status(grpc::StatusCode::INTERNAL, - std::string(status.message())); - } - return grpc::Status::OK; + return ConvertStatus(impl_->Click(request, response)); } grpc::Status Type(grpc::ServerContext* context, const TypeRequest* request, TypeResponse* response) override { - auto status = impl_->Type(request, response); - if (!status.ok()) { - return grpc::Status(grpc::StatusCode::INTERNAL, - std::string(status.message())); - } - return grpc::Status::OK; + return ConvertStatus(impl_->Type(request, response)); } grpc::Status Wait(grpc::ServerContext* context, const WaitRequest* request, WaitResponse* response) override { - auto status = impl_->Wait(request, response); - if (!status.ok()) { - return grpc::Status(grpc::StatusCode::INTERNAL, - std::string(status.message())); - } - return grpc::Status::OK; + return ConvertStatus(impl_->Wait(request, response)); } grpc::Status Assert(grpc::ServerContext* context, const AssertRequest* request, AssertResponse* response) override { - auto status = impl_->Assert(request, response); - if (!status.ok()) { - return grpc::Status(grpc::StatusCode::INTERNAL, - std::string(status.message())); - } - return grpc::Status::OK; + return ConvertStatus(impl_->Assert(request, response)); } grpc::Status Screenshot(grpc::ServerContext* context, const ScreenshotRequest* request, ScreenshotResponse* response) override { - auto status = impl_->Screenshot(request, response); - if (!status.ok()) { - return grpc::Status(grpc::StatusCode::INTERNAL, - std::string(status.message())); - } - return grpc::Status::OK; + return ConvertStatus(impl_->Screenshot(request, response)); + } + + grpc::Status GetTestStatus(grpc::ServerContext* context, + const GetTestStatusRequest* request, + GetTestStatusResponse* response) override { + return ConvertStatus(impl_->GetTestStatus(request, response)); + } + + grpc::Status ListTests(grpc::ServerContext* context, + const ListTestsRequest* request, + ListTestsResponse* response) override { + return ConvertStatus(impl_->ListTests(request, response)); + } + + grpc::Status GetTestResults(grpc::ServerContext* context, + const GetTestResultsRequest* request, + GetTestResultsResponse* response) override { + return ConvertStatus(impl_->GetTestResults(request, response)); } private: + static grpc::Status ConvertStatus(const absl::Status& status) { + if (status.ok()) { + return grpc::Status::OK; + } + + grpc::StatusCode code = grpc::StatusCode::UNKNOWN; + switch (status.code()) { + case absl::StatusCode::kCancelled: + code = grpc::StatusCode::CANCELLED; + break; + case absl::StatusCode::kUnknown: + code = grpc::StatusCode::UNKNOWN; + break; + case absl::StatusCode::kInvalidArgument: + code = grpc::StatusCode::INVALID_ARGUMENT; + break; + case absl::StatusCode::kDeadlineExceeded: + code = grpc::StatusCode::DEADLINE_EXCEEDED; + break; + case absl::StatusCode::kNotFound: + code = grpc::StatusCode::NOT_FOUND; + break; + case absl::StatusCode::kAlreadyExists: + code = grpc::StatusCode::ALREADY_EXISTS; + break; + case absl::StatusCode::kPermissionDenied: + code = grpc::StatusCode::PERMISSION_DENIED; + break; + case absl::StatusCode::kResourceExhausted: + code = grpc::StatusCode::RESOURCE_EXHAUSTED; + break; + case absl::StatusCode::kFailedPrecondition: + code = grpc::StatusCode::FAILED_PRECONDITION; + break; + case absl::StatusCode::kAborted: + code = grpc::StatusCode::ABORTED; + break; + case absl::StatusCode::kOutOfRange: + code = grpc::StatusCode::OUT_OF_RANGE; + break; + case absl::StatusCode::kUnimplemented: + code = grpc::StatusCode::UNIMPLEMENTED; + break; + case absl::StatusCode::kInternal: + code = grpc::StatusCode::INTERNAL; + break; + case absl::StatusCode::kUnavailable: + code = grpc::StatusCode::UNAVAILABLE; + break; + case absl::StatusCode::kDataLoss: + code = grpc::StatusCode::DATA_LOSS; + break; + case absl::StatusCode::kUnauthenticated: + code = grpc::StatusCode::UNAUTHENTICATED; + break; + default: + code = grpc::StatusCode::UNKNOWN; + break; + } + + return grpc::Status(code, std::string(status.message())); + } + ImGuiTestHarnessServiceImpl* impl_; }; @@ -163,117 +274,152 @@ absl::Status ImGuiTestHarnessServiceImpl::Click(const ClickRequest* request, ClickResponse* response) { auto start = std::chrono::steady_clock::now(); -#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE - // Validate test manager if (!test_manager_) { response->set_success(false); response->set_message("TestManager not available"); - return absl::OkStatus(); + response->set_execution_time_ms(0); + return absl::FailedPreconditionError("TestManager not available"); } - // Get ImGuiTestEngine + const std::string test_id = test_manager_->RegisterHarnessTest( + absl::StrFormat("Click: %s", request->target()), "grpc"); + response->set_test_id(test_id); + test_manager_->AppendHarnessTestLog( + test_id, absl::StrCat("Queued click request: ", request->target())); + +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE ImGuiTestEngine* engine = test_manager_->GetUITestEngine(); if (!engine) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = "ImGuiTestEngine not initialized"; response->set_success(false); - response->set_message("ImGuiTestEngine not initialized"); + response->set_message(message); + response->set_execution_time_ms(elapsed.count()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); return absl::OkStatus(); } - // Parse target: "button:Open ROM" -> type=button, label="Open ROM" std::string target = request->target(); size_t colon_pos = target.find(':'); - if (colon_pos == std::string::npos) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = + "Invalid target format. Use 'type:label' (e.g. 'button:Open ROM')"; response->set_success(false); - response->set_message("Invalid target format. Use 'type:label' (e.g. 'button:Open ROM')"); + response->set_message(message); + response->set_execution_time_ms(elapsed.count()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } std::string widget_type = target.substr(0, colon_pos); std::string widget_label = target.substr(colon_pos + 1); - // Convert click type ImGuiMouseButton mouse_button = ImGuiMouseButton_Left; switch (request->type()) { - case ClickRequest::LEFT: + case ClickRequest::CLICK_TYPE_UNSPECIFIED: + case ClickRequest::CLICK_TYPE_LEFT: mouse_button = ImGuiMouseButton_Left; break; - case ClickRequest::RIGHT: + case ClickRequest::CLICK_TYPE_RIGHT: mouse_button = ImGuiMouseButton_Right; break; - case ClickRequest::MIDDLE: + case ClickRequest::CLICK_TYPE_MIDDLE: mouse_button = ImGuiMouseButton_Middle; break; - case ClickRequest::DOUBLE: - // Double click handled below - break; - default: + case ClickRequest::CLICK_TYPE_DOUBLE: + // handled below break; } - // Create a dynamic test to perform the click - auto rpc_state = std::make_shared>(); - auto test_data = std::make_shared(); - test_data->test_func = [=](ImGuiTestContext* ctx) { + TestManager* manager = test_manager_; + test_data->test_func = [manager, captured_id = test_id, widget_type, + widget_label, click_type = request->type(), + mouse_button](ImGuiTestContext* ctx) { + manager->MarkHarnessTestRunning(captured_id); try { - if (request->type() == ClickRequest::DOUBLE) { + if (click_type == ClickRequest::CLICK_TYPE_DOUBLE) { ctx->ItemDoubleClick(widget_label.c_str()); } else { ctx->ItemClick(widget_label.c_str(), mouse_button); } - ctx->Yield(); // Allow UI to process the click before returning - rpc_state->SetResult(true, absl::StrFormat("Clicked %s '%s'", widget_type, widget_label)); + ctx->Yield(); + const std::string success_message = + absl::StrFormat("Clicked %s '%s'", widget_type, widget_label); + manager->AppendHarnessTestLog(captured_id, success_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kPassed, + success_message); } catch (const std::exception& e) { - rpc_state->SetResult(false, absl::StrFormat("Click failed: %s", e.what())); + const std::string error_message = + absl::StrFormat("Click failed: %s", e.what()); + manager->AppendHarnessTestLog(captured_id, error_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kFailed, + error_message); } }; - // Register the test - std::string test_name = absl::StrFormat("grpc_click_%lld", - std::chrono::system_clock::now().time_since_epoch().count()); - + std::string test_name = absl::StrFormat( + "grpc_click_%lld", + static_cast( + std::chrono::system_clock::now().time_since_epoch().count())); + ImGuiTest* test = IM_REGISTER_TEST(engine, "grpc", test_name.c_str()); test->TestFunc = RunDynamicTest; test->UserData = test_data.get(); - - // Queue test for async execution + ImGuiTestEngine_QueueTest(engine, test, ImGuiTestRunFlags_RunFromGui); + KeepDynamicTestData(test_data); - // The test now runs asynchronously. The gRPC call returns immediately. - // The client is responsible for handling the async nature of this operation. - // For now, we'll return a success message indicating the test was queued. - bool success = true; - std::string message = absl::StrFormat("Queued click on %s '%s'", widget_type, widget_label); - - // Note: Test cleanup will be handled by ImGuiTestEngine's FinishTests() - // Do NOT call ImGuiTestEngine_UnregisterTest() here - it causes assertion failure + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = + absl::StrFormat("Queued click on %s '%s'", widget_type, widget_label); + response->set_success(true); + response->set_message(message); + response->set_execution_time_ms(elapsed.count()); + test_manager_->AppendHarnessTestLog(test_id, message); #else - // ImGuiTestEngine not available - stub implementation std::string target = request->target(); size_t colon_pos = target.find(':'); - if (colon_pos == std::string::npos) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = "Invalid target format. Use 'type:label'"; response->set_success(false); - response->set_message("Invalid target format. Use 'type:label'"); + response->set_message(message); + response->set_execution_time_ms(elapsed.count()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } std::string widget_type = target.substr(0, colon_pos); std::string widget_label = target.substr(colon_pos + 1); - bool success = true; - std::string message = absl::StrFormat("[STUB] Clicked %s '%s' (ImGuiTestEngine not available)", - widget_type, widget_label); -#endif + std::string message = absl::StrFormat( + "[STUB] Clicked %s '%s' (ImGuiTestEngine not available)", + widget_type, widget_label); + + test_manager_->MarkHarnessTestRunning(test_id); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->AppendHarnessTestLog(test_id, message); - // Calculate execution time auto elapsed = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); - - response->set_success(success); + response->set_success(true); response->set_message(message); response->set_execution_time_ms(elapsed.count()); +#endif return absl::OkStatus(); } @@ -282,29 +428,46 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, TypeResponse* response) { auto start = std::chrono::steady_clock::now(); -#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE - // Validate test manager if (!test_manager_) { response->set_success(false); response->set_message("TestManager not available"); - return absl::OkStatus(); + response->set_execution_time_ms(0); + return absl::FailedPreconditionError("TestManager not available"); } - // Get ImGuiTestEngine + const std::string test_id = test_manager_->RegisterHarnessTest( + absl::StrFormat("Type: %s", request->target()), "grpc"); + response->set_test_id(test_id); + test_manager_->AppendHarnessTestLog( + test_id, absl::StrFormat("Queued type request: %s", request->target())); + +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE ImGuiTestEngine* engine = test_manager_->GetUITestEngine(); if (!engine) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = "ImGuiTestEngine not initialized"; response->set_success(false); - response->set_message("ImGuiTestEngine not initialized"); + response->set_message(message); + response->set_execution_time_ms(elapsed.count()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); return absl::OkStatus(); } - // Parse target: "input:Filename" -> type=input, label="Filename" std::string target = request->target(); size_t colon_pos = target.find(':'); - if (colon_pos == std::string::npos) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = + "Invalid target format. Use 'type:label' (e.g. 'input:Filename')"; response->set_success(false); - response->set_message("Invalid target format. Use 'type:label' (e.g. 'input:Filename')"); + response->set_message(message); + response->set_execution_time_ms(elapsed.count()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } @@ -313,84 +476,108 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, std::string text = request->text(); bool clear_first = request->clear_first(); - // Create a dynamic test to perform the typing auto rpc_state = std::make_shared>(); - auto test_data = std::make_shared(); - test_data->test_func = [=](ImGuiTestContext* ctx) { + TestManager* manager = test_manager_; + test_data->test_func = [manager, captured_id = test_id, widget_type, + widget_label, clear_first, text]( + ImGuiTestContext* ctx) { + manager->MarkHarnessTestRunning(captured_id); try { - // Find the input field ImGuiTestItemInfo item = ctx->ItemInfo(widget_label.c_str()); if (item.ID == 0) { - rpc_state->SetResult(false, absl::StrFormat("Input field '%s' not found", widget_label)); + std::string error_message = + absl::StrFormat("Input field '%s' not found", widget_label); + manager->AppendHarnessTestLog(captured_id, error_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kFailed, + error_message); + rpc_state->SetResult(false, error_message); return; } - // Click to focus the input field first ctx->ItemClick(widget_label.c_str()); - - // Clear existing text if requested if (clear_first) { - // Select all (Ctrl+A or Cmd+A depending on platform) ctx->KeyPress(ImGuiMod_Shortcut | ImGuiKey_A); - // Delete selected text ctx->KeyPress(ImGuiKey_Delete); } - // Type the new text ctx->ItemInputValue(widget_label.c_str(), text.c_str()); - - rpc_state->SetResult(true, absl::StrFormat("Typed '%s' into %s '%s'%s", - text, widget_type, widget_label, - clear_first ? " (cleared first)" : "")); + + std::string success_message = absl::StrFormat( + "Typed '%s' into %s '%s'%s", text, widget_type, widget_label, + clear_first ? " (cleared first)" : ""); + manager->AppendHarnessTestLog(captured_id, success_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kPassed, + success_message); + rpc_state->SetResult(true, success_message); } catch (const std::exception& e) { - rpc_state->SetResult(false, absl::StrFormat("Type failed: %s", e.what())); + std::string error_message = + absl::StrFormat("Type failed: %s", e.what()); + manager->AppendHarnessTestLog(captured_id, error_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kFailed, + error_message); + rpc_state->SetResult(false, error_message); } }; - // Register the test - std::string test_name = absl::StrFormat("grpc_type_%lld", - std::chrono::system_clock::now().time_since_epoch().count()); - + std::string test_name = absl::StrFormat( + "grpc_type_%lld", + static_cast( + std::chrono::system_clock::now().time_since_epoch().count())); + ImGuiTest* test = IM_REGISTER_TEST(engine, "grpc", test_name.c_str()); test->TestFunc = RunDynamicTest; test->UserData = test_data.get(); - - // Queue test for async execution + ImGuiTestEngine_QueueTest(engine, test, ImGuiTestRunFlags_RunFromGui); - - // Poll for test completion (with timeout) + KeepDynamicTestData(test_data); + auto timeout = std::chrono::seconds(5); auto wait_start = std::chrono::steady_clock::now(); while (!rpc_state->completed.load()) { if (std::chrono::steady_clock::now() - wait_start > timeout) { - rpc_state->SetResult(false, "Test timeout - input field not found or unresponsive"); + std::string error_message = + "Test timeout - input field not found or unresponsive"; + manager->AppendHarnessTestLog(test_id, error_message); + manager->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kTimeout, error_message); + rpc_state->SetResult(false, error_message); break; } - // Yield to allow ImGui event processing std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - - bool success; + + bool success = false; std::string message; rpc_state->GetResult(success, message); - - // Note: Test cleanup will be handled by ImGuiTestEngine's FinishTests() - // Do NOT call ImGuiTestEngine_UnregisterTest() here - it causes assertion failure - -#else - // ImGuiTestEngine not available - stub implementation - bool success = true; - std::string message = absl::StrFormat("[STUB] Typed '%s' into %s (ImGuiTestEngine not available)", - request->text(), request->target()); -#endif - auto elapsed = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); response->set_success(success); response->set_message(message); response->set_execution_time_ms(elapsed.count()); + if (!message.empty()) { + test_manager_->AppendHarnessTestLog(test_id, message); + } + +#else + test_manager_->MarkHarnessTestRunning(test_id); + std::string message = absl::StrFormat( + "[STUB] Typed '%s' into %s (ImGuiTestEngine not available)", + request->text(), request->target()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->AppendHarnessTestLog(test_id, message); + + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + response->set_success(true); + response->set_message(message); + response->set_execution_time_ms(elapsed.count()); +#endif return absl::OkStatus(); } @@ -399,307 +586,365 @@ absl::Status ImGuiTestHarnessServiceImpl::Wait(const WaitRequest* request, WaitResponse* response) { auto start = std::chrono::steady_clock::now(); -#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE - // Validate test manager if (!test_manager_) { response->set_success(false); response->set_message("TestManager not available"); - return absl::OkStatus(); + response->set_elapsed_ms(0); + return absl::FailedPreconditionError("TestManager not available"); } - // Get ImGuiTestEngine + const std::string test_id = test_manager_->RegisterHarnessTest( + absl::StrFormat("Wait: %s", request->condition()), "grpc"); + response->set_test_id(test_id); + test_manager_->AppendHarnessTestLog( + test_id, absl::StrFormat("Queued wait condition: %s", + request->condition())); + +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE ImGuiTestEngine* engine = test_manager_->GetUITestEngine(); if (!engine) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = "ImGuiTestEngine not initialized"; response->set_success(false); - response->set_message("ImGuiTestEngine not initialized"); + response->set_message(message); + response->set_elapsed_ms(elapsed.count()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); return absl::OkStatus(); } - // Parse condition: "window_visible:Overworld Editor" -> check if window is visible std::string condition = request->condition(); size_t colon_pos = condition.find(':'); - if (colon_pos == std::string::npos) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + std::string message = + "Invalid condition format. Use 'type:target' (e.g. 'window_visible:Overworld Editor')"; response->set_success(false); - response->set_message("Invalid condition format. Use 'type:target' (e.g. 'window_visible:Overworld Editor')"); + response->set_message(message); + response->set_elapsed_ms(elapsed.count()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } std::string condition_type = condition.substr(0, colon_pos); std::string condition_target = condition.substr(colon_pos + 1); - - int timeout_ms = request->timeout_ms() > 0 ? request->timeout_ms() : 5000; // Default 5s - int poll_interval_ms = request->poll_interval_ms() > 0 ? request->poll_interval_ms() : 100; // Default 100ms + int timeout_ms = request->timeout_ms() > 0 ? request->timeout_ms() : 5000; + int poll_interval_ms = request->poll_interval_ms() > 0 + ? request->poll_interval_ms() + : 100; - // Create thread-safe shared state for communication - auto rpc_state = std::make_shared>(); - auto test_data = std::make_shared(); - test_data->test_func = [rpc_state, condition_type, condition_target, - timeout_ms, poll_interval_ms](ImGuiTestContext* ctx) { + TestManager* manager = test_manager_; + test_data->test_func = [manager, captured_id = test_id, condition_type, + condition_target, timeout_ms, poll_interval_ms]( + ImGuiTestContext* ctx) { + manager->MarkHarnessTestRunning(captured_id); + auto poll_start = std::chrono::steady_clock::now(); + auto timeout = std::chrono::milliseconds(timeout_ms); + + for (int i = 0; i < 10; ++i) { + ctx->Yield(); + } + try { - auto poll_start = std::chrono::steady_clock::now(); - auto timeout = std::chrono::milliseconds(timeout_ms); - - // Give ImGui time to process the menu click and create windows - for (int i = 0; i < 10; i++) { - ctx->Yield(); - } - while (std::chrono::steady_clock::now() - poll_start < timeout) { bool current_state = false; - // Check the condition type using thread-safe ctx methods if (condition_type == "window_visible") { - // Use ctx->WindowInfo instead of ImGui::FindWindowByName for thread safety - ImGuiTestItemInfo window_info = ctx->WindowInfo(condition_target.c_str(), - ImGuiTestOpFlags_NoError); + ImGuiTestItemInfo window_info = ctx->WindowInfo( + condition_target.c_str(), ImGuiTestOpFlags_NoError); current_state = (window_info.ID != 0); } else if (condition_type == "element_visible") { - ImGuiTestItemInfo item = ctx->ItemInfo(condition_target.c_str()); - current_state = (item.ID != 0 && item.RectClipped.GetWidth() > 0 && - item.RectClipped.GetHeight() > 0); + ImGuiTestItemInfo item = ctx->ItemInfo( + condition_target.c_str(), ImGuiTestOpFlags_NoError); + current_state = (item.ID != 0 && item.RectClipped.GetWidth() > 0 && + item.RectClipped.GetHeight() > 0); } else if (condition_type == "element_enabled") { - ImGuiTestItemInfo item = ctx->ItemInfo(condition_target.c_str()); - current_state = (item.ID != 0 && !(item.ItemFlags & ImGuiItemFlags_Disabled)); + ImGuiTestItemInfo item = ctx->ItemInfo( + condition_target.c_str(), ImGuiTestOpFlags_NoError); + current_state = + (item.ID != 0 && !(item.ItemFlags & ImGuiItemFlags_Disabled)); } else { - rpc_state->SetResult(false, absl::StrFormat("Unknown condition type: %s", condition_type)); + std::string error_message = + absl::StrFormat("Unknown condition type: %s", condition_type); + manager->AppendHarnessTestLog(captured_id, error_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kFailed, + error_message); return; } if (current_state) { - rpc_state->SetResult(true, absl::StrFormat("Condition '%s:%s' met after %lld ms", - condition_type, condition_target, - std::chrono::duration_cast( - std::chrono::steady_clock::now() - poll_start).count())); + auto elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - poll_start); + std::string success_message = absl::StrFormat( + "Condition '%s:%s' met after %lld ms", condition_type, + condition_target, static_cast(elapsed_ms.count())); + manager->AppendHarnessTestLog(captured_id, success_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kPassed, + success_message); return; } - // Sleep before next poll - std::this_thread::sleep_for(std::chrono::milliseconds(poll_interval_ms)); - ctx->Yield(); // Let ImGui process events + std::this_thread::sleep_for( + std::chrono::milliseconds(poll_interval_ms)); + ctx->Yield(); } - // Timeout reached - rpc_state->SetResult(false, absl::StrFormat("Condition '%s:%s' not met after %d ms timeout", - condition_type, condition_target, timeout_ms)); + std::string timeout_message = absl::StrFormat( + "Condition '%s:%s' not met after %d ms timeout", condition_type, + condition_target, timeout_ms); + manager->AppendHarnessTestLog(captured_id, timeout_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kTimeout, + timeout_message); } catch (const std::exception& e) { - rpc_state->SetResult(false, absl::StrFormat("Wait failed: %s", e.what())); + std::string error_message = + absl::StrFormat("Wait failed: %s", e.what()); + manager->AppendHarnessTestLog(captured_id, error_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kFailed, + error_message); } }; - // Register the test - std::string test_name = absl::StrFormat("grpc_wait_%lld", - std::chrono::system_clock::now().time_since_epoch().count()); - + std::string test_name = absl::StrFormat( + "grpc_wait_%lld", + static_cast( + std::chrono::system_clock::now().time_since_epoch().count())); + ImGuiTest* test = IM_REGISTER_TEST(engine, "grpc", test_name.c_str()); test->TestFunc = RunDynamicTest; test->UserData = test_data.get(); - - // Queue test for async execution + ImGuiTestEngine_QueueTest(engine, test, ImGuiTestRunFlags_RunFromGui); - // The test now runs asynchronously. The gRPC call returns immediately. - bool condition_met = true; // Assume it will be met - std::string message = absl::StrFormat("Queued wait for '%s:%s'", condition_type, condition_target); - - // Note: Test cleanup will be handled by ImGuiTestEngine's FinishTests() - // Do NOT call ImGuiTestEngine_UnregisterTest() here - it causes assertion failure - -#else - // ImGuiTestEngine not available - stub implementation - bool condition_met = true; - std::string message = absl::StrFormat("[STUB] Condition '%s' met (ImGuiTestEngine not available)", - request->condition()); -#endif - auto elapsed = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); - - response->set_success(condition_met); + std::string message = + absl::StrFormat("Queued wait for '%s:%s'", condition_type, + condition_target); + response->set_success(true); response->set_message(message); response->set_elapsed_ms(elapsed.count()); + test_manager_->AppendHarnessTestLog(test_id, message); + +#else + test_manager_->MarkHarnessTestRunning(test_id); + std::string message = absl::StrFormat( + "[STUB] Condition '%s' met (ImGuiTestEngine not available)", + request->condition()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->AppendHarnessTestLog(test_id, message); + + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + response->set_success(true); + response->set_message(message); + response->set_elapsed_ms(elapsed.count()); +#endif return absl::OkStatus(); } absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request, AssertResponse* response) { -#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE - // Validate test manager if (!test_manager_) { response->set_success(false); response->set_message("TestManager not available"); response->set_actual_value("N/A"); response->set_expected_value("N/A"); - return absl::OkStatus(); + return absl::FailedPreconditionError("TestManager not available"); } - // Get ImGuiTestEngine + const std::string test_id = test_manager_->RegisterHarnessTest( + absl::StrFormat("Assert: %s", request->condition()), "grpc"); + response->set_test_id(test_id); + test_manager_->AppendHarnessTestLog( + test_id, absl::StrFormat("Queued assertion: %s", request->condition())); + +#if defined(YAZE_ENABLE_IMGUI_TEST_ENGINE) && YAZE_ENABLE_IMGUI_TEST_ENGINE ImGuiTestEngine* engine = test_manager_->GetUITestEngine(); if (!engine) { + std::string message = "ImGuiTestEngine not initialized"; response->set_success(false); - response->set_message("ImGuiTestEngine not initialized"); + response->set_message(message); response->set_actual_value("N/A"); response->set_expected_value("N/A"); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); return absl::OkStatus(); } - // Parse condition: "visible:Main Window" -> check if element is visible std::string condition = request->condition(); size_t colon_pos = condition.find(':'); - if (colon_pos == std::string::npos) { + std::string message = + "Invalid condition format. Use 'type:target' (e.g. 'visible:Main Window')"; response->set_success(false); - response->set_message("Invalid condition format. Use 'type:target' (e.g. 'visible:Main Window')"); + response->set_message(message); response->set_actual_value("N/A"); response->set_expected_value("N/A"); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } std::string assertion_type = condition.substr(0, colon_pos); std::string assertion_target = condition.substr(colon_pos + 1); - struct AssertResult { - bool passed; - std::string message; - std::string actual_value; - std::string expected_value; - }; - - // Create thread-safe shared state for communication - auto rpc_state = std::make_shared>(); - auto test_data = std::make_shared(); - test_data->test_func = [rpc_state, assertion_type, assertion_target](ImGuiTestContext* ctx) { + TestManager* manager = test_manager_; + test_data->test_func = [manager, captured_id = test_id, assertion_type, + assertion_target](ImGuiTestContext* ctx) { + manager->MarkHarnessTestRunning(captured_id); + + auto complete_with = + [manager, captured_id](bool passed, const std::string& message, + const std::string& actual, + const std::string& expected, + TestManager::HarnessTestStatus status) { + manager->AppendHarnessTestLog(captured_id, message); + if (!actual.empty() || !expected.empty()) { + manager->AppendHarnessTestLog( + captured_id, + absl::StrFormat("Actual: %s | Expected: %s", actual, + expected)); + } + manager->MarkHarnessTestCompleted( + captured_id, status, + passed ? "" : message); + }; + try { - AssertResult result; - + bool passed = false; + std::string actual_value; + std::string expected_value; + std::string message; + if (assertion_type == "visible") { - // Check if window is visible using thread-safe context - ImGuiTestItemInfo window_info = ctx->WindowInfo(assertion_target.c_str(), ImGuiTestOpFlags_NoError); + ImGuiTestItemInfo window_info = ctx->WindowInfo( + assertion_target.c_str(), ImGuiTestOpFlags_NoError); bool is_visible = (window_info.ID != 0); - - result.passed = is_visible; - result.actual_value = is_visible ? "visible" : "hidden"; - result.expected_value = "visible"; - result.message = result.passed - ? absl::StrFormat("'%s' is visible", assertion_target) - : absl::StrFormat("'%s' is not visible", assertion_target); - + passed = is_visible; + actual_value = is_visible ? "visible" : "hidden"; + expected_value = "visible"; + message = passed ? + absl::StrFormat("'%s' is visible", assertion_target) : + absl::StrFormat("'%s' is not visible", assertion_target); } else if (assertion_type == "enabled") { - // Check if element is enabled - ImGuiTestItemInfo item = ctx->ItemInfo(assertion_target.c_str(), ImGuiTestOpFlags_NoError); - bool is_enabled = (item.ID != 0 && !(item.ItemFlags & ImGuiItemFlags_Disabled)); - - result.passed = is_enabled; - result.actual_value = is_enabled ? "enabled" : "disabled"; - result.expected_value = "enabled"; - result.message = result.passed - ? absl::StrFormat("'%s' is enabled", assertion_target) - : absl::StrFormat("'%s' is not enabled", assertion_target); - + ImGuiTestItemInfo item = ctx->ItemInfo( + assertion_target.c_str(), ImGuiTestOpFlags_NoError); + bool is_enabled = (item.ID != 0 && + !(item.ItemFlags & ImGuiItemFlags_Disabled)); + passed = is_enabled; + actual_value = is_enabled ? "enabled" : "disabled"; + expected_value = "enabled"; + message = passed ? + absl::StrFormat("'%s' is enabled", assertion_target) : + absl::StrFormat("'%s' is not enabled", assertion_target); } else if (assertion_type == "exists") { - // Check if element exists - ImGuiTestItemInfo item = ctx->ItemInfo(assertion_target.c_str(), ImGuiTestOpFlags_NoError); + ImGuiTestItemInfo item = ctx->ItemInfo( + assertion_target.c_str(), ImGuiTestOpFlags_NoError); bool exists = (item.ID != 0); - - result.passed = exists; - result.actual_value = exists ? "exists" : "not found"; - result.expected_value = "exists"; - result.message = result.passed - ? absl::StrFormat("'%s' exists", assertion_target) - : absl::StrFormat("'%s' not found", assertion_target); - + passed = exists; + actual_value = exists ? "exists" : "not found"; + expected_value = "exists"; + message = passed ? + absl::StrFormat("'%s' exists", assertion_target) : + absl::StrFormat("'%s' not found", assertion_target); } else if (assertion_type == "text_contains") { - // Check if text input contains expected text (requires expected_value in condition) - // Format: "text_contains:MyInput:ExpectedText" size_t second_colon = assertion_target.find(':'); if (second_colon == std::string::npos) { - result.passed = false; - result.message = "text_contains requires format 'text_contains:target:expected_text'"; - result.actual_value = "N/A"; - result.expected_value = "N/A"; - rpc_state->SetResult(result, result.message); + std::string error_message = + "text_contains requires format 'text_contains:target:expected_text'"; + complete_with(false, error_message, "N/A", "N/A", + TestManager::HarnessTestStatus::kFailed); return; } - + std::string input_target = assertion_target.substr(0, second_colon); std::string expected_text = assertion_target.substr(second_colon + 1); - + ImGuiTestItemInfo item = ctx->ItemInfo(input_target.c_str()); if (item.ID != 0) { - // Note: Text retrieval is simplified - actual implementation may need widget-specific handling std::string actual_text = "(text_retrieval_not_fully_implemented)"; - - result.passed = (actual_text.find(expected_text) != std::string::npos); - result.actual_value = actual_text; - result.expected_value = absl::StrFormat("contains '%s'", expected_text); - result.message = result.passed - ? absl::StrFormat("'%s' contains '%s'", input_target, expected_text) - : absl::StrFormat("'%s' does not contain '%s' (actual: '%s')", - input_target, expected_text, actual_text); + passed = actual_text.find(expected_text) != std::string::npos; + actual_value = actual_text; + expected_value = absl::StrFormat("contains '%s'", expected_text); + message = passed ? + absl::StrFormat("'%s' contains '%s'", input_target, + expected_text) : + absl::StrFormat( + "'%s' does not contain '%s' (actual: '%s')", + input_target, expected_text, actual_text); } else { - result.passed = false; - result.message = absl::StrFormat("Input '%s' not found", input_target); - result.actual_value = "not found"; - result.expected_value = expected_text; + passed = false; + actual_value = "not found"; + expected_value = expected_text; + message = absl::StrFormat("Input '%s' not found", input_target); } - } else { - result.passed = false; - result.message = absl::StrFormat("Unknown assertion type: %s", assertion_type); - result.actual_value = "N/A"; - result.expected_value = "N/A"; + std::string error_message = + absl::StrFormat("Unknown assertion type: %s", assertion_type); + complete_with(false, error_message, "N/A", "N/A", + TestManager::HarnessTestStatus::kFailed); + return; } - - // Store result in thread-safe state - rpc_state->SetResult(result, result.message); - + + complete_with(passed, message, actual_value, expected_value, + passed ? TestManager::HarnessTestStatus::kPassed + : TestManager::HarnessTestStatus::kFailed); } catch (const std::exception& e) { - AssertResult result; - result.passed = false; - result.message = absl::StrFormat("Assertion failed: %s", e.what()); - result.actual_value = "exception"; - result.expected_value = "N/A"; - rpc_state->SetResult(result, result.message); + std::string error_message = + absl::StrFormat("Assertion failed: %s", e.what()); + manager->AppendHarnessTestLog(captured_id, error_message); + manager->MarkHarnessTestCompleted( + captured_id, TestManager::HarnessTestStatus::kFailed, + error_message); } }; - // Register the test - std::string test_name = absl::StrFormat("grpc_assert_%lld", - std::chrono::system_clock::now().time_since_epoch().count()); - + std::string test_name = absl::StrFormat( + "grpc_assert_%lld", + static_cast( + std::chrono::system_clock::now().time_since_epoch().count())); + ImGuiTest* test = IM_REGISTER_TEST(engine, "grpc", test_name.c_str()); test->TestFunc = RunDynamicTest; test->UserData = test_data.get(); - - // Queue test for async execution + ImGuiTestEngine_QueueTest(engine, test, ImGuiTestRunFlags_RunFromGui); - // The test now runs asynchronously. The gRPC call returns immediately. - AssertResult final_result; - final_result.passed = true; // Assume pass - final_result.message = absl::StrFormat("Queued assertion for '%s:%s'", assertion_type, assertion_target); - final_result.actual_value = "(async)"; - final_result.expected_value = "(async)"; - - // Note: Test cleanup will be handled by ImGuiTestEngine's FinishTests() - // Do NOT call ImGuiTestEngine_UnregisterTest() here - it causes assertion failure + response->set_success(true); + std::string message = absl::StrFormat( + "Queued assertion for '%s:%s'", assertion_type, assertion_target); + response->set_message(message); + response->set_actual_value("(async)"); + response->set_expected_value("(async)"); + test_manager_->AppendHarnessTestLog(test_id, message); #else - // ImGuiTestEngine not available - stub implementation - bool assertion_passed = true; - std::string message = absl::StrFormat("[STUB] Assertion '%s' passed (ImGuiTestEngine not available)", - request->condition()); - std::string actual_value = "(stub)"; - std::string expected_value = "(stub)"; -#endif + test_manager_->MarkHarnessTestRunning(test_id); + std::string message = absl::StrFormat( + "[STUB] Assertion '%s' passed (ImGuiTestEngine not available)", + request->condition()); + test_manager_->MarkHarnessTestCompleted( + test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->AppendHarnessTestLog(test_id, message); - response->set_success(final_result.passed); - response->set_message(final_result.message); - response->set_actual_value(final_result.actual_value); - response->set_expected_value(final_result.expected_value); + response->set_success(true); + response->set_message(message); + response->set_actual_value("(stub)"); + response->set_expected_value("(stub)"); +#endif return absl::OkStatus(); } @@ -716,6 +961,187 @@ absl::Status ImGuiTestHarnessServiceImpl::Screenshot( return absl::OkStatus(); } +absl::Status ImGuiTestHarnessServiceImpl::GetTestStatus( + const GetTestStatusRequest* request, GetTestStatusResponse* response) { + if (!test_manager_) { + return absl::FailedPreconditionError("TestManager not available"); + } + + if (request->test_id().empty()) { + return absl::InvalidArgumentError("test_id must be provided"); + } + + auto execution_or = + test_manager_->GetHarnessTestExecution(request->test_id()); + if (!execution_or.ok()) { + response->set_status(GetTestStatusResponse::STATUS_UNSPECIFIED); + response->set_error_message(std::string(execution_or.status().message())); + return absl::OkStatus(); + } + + const auto& execution = execution_or.value(); + response->set_status(ConvertHarnessStatus(execution.status)); + response->set_queued_at_ms(ToUnixMillisSafe(execution.queued_at)); + response->set_started_at_ms(ToUnixMillisSafe(execution.started_at)); + response->set_completed_at_ms(ToUnixMillisSafe(execution.completed_at)); + response->set_execution_time_ms(ClampDurationToInt32(execution.duration)); + if (!execution.error_message.empty()) { + response->set_error_message(execution.error_message); + } else { + response->clear_error_message(); + } + + response->clear_assertion_failures(); + for (const auto& failure : execution.assertion_failures) { + response->add_assertion_failures(failure); + } + + return absl::OkStatus(); +} + +absl::Status ImGuiTestHarnessServiceImpl::ListTests( + const ListTestsRequest* request, ListTestsResponse* response) { + if (!test_manager_) { + return absl::FailedPreconditionError("TestManager not available"); + } + + if (request->page_size() < 0) { + return absl::InvalidArgumentError("page_size cannot be negative"); + } + + int page_size = request->page_size() > 0 ? request->page_size() : 100; + constexpr int kMaxPageSize = 500; + if (page_size > kMaxPageSize) { + page_size = kMaxPageSize; + } + + size_t start_index = 0; + if (!request->page_token().empty()) { + int64_t token_value = 0; + if (!absl::SimpleAtoi(request->page_token(), &token_value) || + token_value < 0) { + return absl::InvalidArgumentError("Invalid page_token"); + } + start_index = static_cast(token_value); + } + + auto summaries = + test_manager_->ListHarnessTestSummaries(request->category_filter()); + + response->set_total_count(static_cast(summaries.size())); + + if (start_index >= summaries.size()) { + response->clear_tests(); + response->clear_next_page_token(); + return absl::OkStatus(); + } + + size_t end_index = + std::min(start_index + static_cast(page_size), + summaries.size()); + + for (size_t i = start_index; i < end_index; ++i) { + const auto& summary = summaries[i]; + auto* test_info = response->add_tests(); + const auto& exec = summary.latest_execution; + + test_info->set_test_id(exec.test_id); + test_info->set_name(exec.name); + test_info->set_category(exec.category); + + int64_t last_run_ms = ToUnixMillisSafe(exec.completed_at); + if (last_run_ms == 0) { + last_run_ms = ToUnixMillisSafe(exec.started_at); + } + if (last_run_ms == 0) { + last_run_ms = ToUnixMillisSafe(exec.queued_at); + } + test_info->set_last_run_timestamp_ms(last_run_ms); + + test_info->set_total_runs(summary.total_runs); + test_info->set_pass_count(summary.pass_count); + test_info->set_fail_count(summary.fail_count); + + int32_t average_duration_ms = 0; + if (summary.total_runs > 0) { + absl::Duration average_duration = + summary.total_duration / summary.total_runs; + average_duration_ms = ClampDurationToInt32(average_duration); + } + test_info->set_average_duration_ms(average_duration_ms); + } + + if (end_index < summaries.size()) { + response->set_next_page_token(absl::StrCat(end_index)); + } else { + response->clear_next_page_token(); + } + + return absl::OkStatus(); +} + +absl::Status ImGuiTestHarnessServiceImpl::GetTestResults( + const GetTestResultsRequest* request, + GetTestResultsResponse* response) { + if (!test_manager_) { + return absl::FailedPreconditionError("TestManager not available"); + } + + if (request->test_id().empty()) { + return absl::InvalidArgumentError("test_id must be provided"); + } + + auto execution_or = + test_manager_->GetHarnessTestExecution(request->test_id()); + if (!execution_or.ok()) { + return execution_or.status(); + } + + const auto& execution = execution_or.value(); + response->set_success( + execution.status == TestManager::HarnessTestStatus::kPassed); + response->set_test_name(execution.name); + response->set_category(execution.category); + + int64_t executed_at_ms = ToUnixMillisSafe(execution.completed_at); + if (executed_at_ms == 0) { + executed_at_ms = ToUnixMillisSafe(execution.started_at); + } + if (executed_at_ms == 0) { + executed_at_ms = ToUnixMillisSafe(execution.queued_at); + } + response->set_executed_at_ms(executed_at_ms); + response->set_duration_ms(ClampDurationToInt32(execution.duration)); + + response->clear_assertions(); + if (!execution.assertion_failures.empty()) { + for (const auto& failure : execution.assertion_failures) { + auto* assertion = response->add_assertions(); + assertion->set_description(failure); + assertion->set_passed(false); + assertion->set_error_message(failure); + } + } else if (!execution.error_message.empty()) { + auto* assertion = response->add_assertions(); + assertion->set_description("Execution error"); + assertion->set_passed(false); + assertion->set_error_message(execution.error_message); + } + + if (request->include_logs()) { + for (const auto& log_entry : execution.logs) { + response->add_logs(log_entry); + } + } + + auto* metrics_map = response->mutable_metrics(); + for (const auto& [key, value] : execution.metrics) { + (*metrics_map)[key] = value; + } + + return absl::OkStatus(); +} + // ============================================================================ // ImGuiTestHarnessServer - Server Lifecycle // ============================================================================ diff --git a/src/app/core/imgui_test_harness_service.h b/src/app/core/imgui_test_harness_service.h index 52eb60b0..b1ac78b2 100644 --- a/src/app/core/imgui_test_harness_service.h +++ b/src/app/core/imgui_test_harness_service.h @@ -36,6 +36,12 @@ class AssertRequest; class AssertResponse; class ScreenshotRequest; class ScreenshotResponse; +class GetTestStatusRequest; +class GetTestStatusResponse; +class ListTestsRequest; +class ListTestsResponse; +class GetTestResultsRequest; +class GetTestResultsResponse; // Implementation of ImGuiTestHarness gRPC service // This class provides the actual RPC handlers for automated GUI testing @@ -72,6 +78,14 @@ class ImGuiTestHarnessServiceImpl { absl::Status Screenshot(const ScreenshotRequest* request, ScreenshotResponse* response); + // Test introspection APIs + absl::Status GetTestStatus(const GetTestStatusRequest* request, + GetTestStatusResponse* response); + absl::Status ListTests(const ListTestsRequest* request, + ListTestsResponse* response); + absl::Status GetTestResults(const GetTestResultsRequest* request, + GetTestResultsResponse* response); + private: TestManager* test_manager_; // Non-owning pointer to access ImGuiTestEngine }; diff --git a/src/app/core/proto/imgui_test_harness.proto b/src/app/core/proto/imgui_test_harness.proto index 9e041d62..66bd0997 100644 --- a/src/app/core/proto/imgui_test_harness.proto +++ b/src/app/core/proto/imgui_test_harness.proto @@ -22,6 +22,11 @@ service ImGuiTestHarness { // Capture a screenshot rpc Screenshot(ScreenshotRequest) returns (ScreenshotResponse); + + // Test introspection APIs (IT-05) + rpc GetTestStatus(GetTestStatusRequest) returns (GetTestStatusResponse); + rpc ListTests(ListTestsRequest) returns (ListTestsResponse); + rpc GetTestResults(GetTestResultsRequest) returns (GetTestResultsResponse); } // ============================================================================ @@ -43,14 +48,15 @@ message PingResponse { // ============================================================================ message ClickRequest { - string target = 1; // Target element (e.g., "button:Open ROM", "menu:File/Open") + string target = 1; // Target element (e.g., "button:Open ROM") ClickType type = 2; // Type of click enum ClickType { - LEFT = 0; // Single left click - RIGHT = 1; // Single right click - DOUBLE = 2; // Double click - MIDDLE = 3; // Middle mouse button + CLICK_TYPE_UNSPECIFIED = 0; // Default/unspecified click type + CLICK_TYPE_LEFT = 1; // Single left click + CLICK_TYPE_RIGHT = 2; // Single right click + CLICK_TYPE_DOUBLE = 3; // Double click + CLICK_TYPE_MIDDLE = 4; // Middle mouse button } } @@ -58,6 +64,7 @@ message ClickResponse { bool success = 1; // Whether the click succeeded string message = 2; // Human-readable result message int32 execution_time_ms = 3; // Time taken to execute (for debugging) + string test_id = 4; // Unique test identifier for introspection } // ============================================================================ @@ -74,6 +81,7 @@ message TypeResponse { bool success = 1; string message = 2; int32 execution_time_ms = 3; + string test_id = 4; } // ============================================================================ @@ -81,7 +89,7 @@ message TypeResponse { // ============================================================================ message WaitRequest { - string condition = 1; // Condition to wait for (e.g., "window:Overworld Editor", "enabled:button:Save") + string condition = 1; // Condition to wait for (e.g., "window:Overworld") int32 timeout_ms = 2; // Maximum time to wait (default 5000ms) int32 poll_interval_ms = 3; // How often to check (default 100ms) } @@ -90,6 +98,7 @@ message WaitResponse { bool success = 1; // Whether condition was met before timeout string message = 2; int32 elapsed_ms = 3; // Time taken before condition met (or timeout) + string test_id = 4; // Unique test identifier for introspection } // ============================================================================ @@ -97,7 +106,7 @@ message WaitResponse { // ============================================================================ message AssertRequest { - string condition = 1; // Condition to assert (e.g., "visible:button:Save", "text:label:Version:0.3.2") + string condition = 1; // Condition to assert (e.g., "visible:button:Save") string failure_message = 2; // Custom message if assertion fails } @@ -106,6 +115,7 @@ message AssertResponse { string message = 2; // Diagnostic message string actual_value = 3; // Actual value found (for debugging) string expected_value = 4; // Expected value (for debugging) + string test_id = 5; // Unique test identifier for introspection } // ============================================================================ @@ -118,8 +128,9 @@ message ScreenshotRequest { ImageFormat format = 3; // Image format enum ImageFormat { - PNG = 0; - JPEG = 1; + IMAGE_FORMAT_UNSPECIFIED = 0; + IMAGE_FORMAT_PNG = 1; + IMAGE_FORMAT_JPEG = 2; } } @@ -129,3 +140,85 @@ message ScreenshotResponse { string file_path = 3; // Absolute path to saved screenshot int64 file_size_bytes = 4; } + +// ============================================================================ +// GetTestStatus - Query test execution state +// ============================================================================ + +message GetTestStatusRequest { + string test_id = 1; // Test ID from Click/Type/Wait/Assert response +} + +message GetTestStatusResponse { + enum Status { + STATUS_UNSPECIFIED = 0; // Test ID not found or unspecified + STATUS_QUEUED = 1; // Waiting to execute + STATUS_RUNNING = 2; // Currently executing + STATUS_PASSED = 3; // Completed successfully + STATUS_FAILED = 4; // Assertion failed or error + STATUS_TIMEOUT = 5; // Exceeded timeout + } + + Status status = 1; + int64 queued_at_ms = 2; // When test was queued + int64 started_at_ms = 3; // When test started (0 if not started) + int64 completed_at_ms = 4; // When test completed (0 if not complete) + int32 execution_time_ms = 5; // Total execution time + string error_message = 6; // Error details if FAILED/TIMEOUT + repeated string assertion_failures = 7; // Failed assertion details +} + +// ============================================================================ +// ListTests - Enumerate available tests +// ============================================================================ + +message ListTestsRequest { + string category_filter = 1; // Optional: "grpc", "unit", "integration", "e2e" + int32 page_size = 2; // Number of results per page (default 100) + string page_token = 3; // Pagination token from previous response +} + +message ListTestsResponse { + repeated TestInfo tests = 1; + string next_page_token = 2; // Token for next page (empty if no more) + int32 total_count = 3; // Total number of matching tests +} + +message TestInfo { + string test_id = 1; // Unique test identifier + string name = 2; // Human-readable test name + string category = 3; // Category: grpc, unit, integration, e2e + int64 last_run_timestamp_ms = 4; // When test last executed + int32 total_runs = 5; // Total number of executions + int32 pass_count = 6; // Number of successful runs + int32 fail_count = 7; // Number of failed runs + int32 average_duration_ms = 8; // Average execution time +} + +// ============================================================================ +// GetTestResults - Retrieve detailed results +// ============================================================================ + +message GetTestResultsRequest { + string test_id = 1; + bool include_logs = 2; // Include full execution logs +} + +message GetTestResultsResponse { + bool success = 1; // Overall test result + string test_name = 2; + string category = 3; + int64 executed_at_ms = 4; + int32 duration_ms = 5; + repeated AssertionResult assertions = 6; + repeated string logs = 7; // If include_logs=true + map metrics = 8; // e.g., "frame_count": 123 +} + +message AssertionResult { + string description = 1; + bool passed = 2; + string expected_value = 3; + string actual_value = 4; + string error_message = 5; +} diff --git a/src/app/test/test_manager.cc b/src/app/test/test_manager.cc index bed471f9..777e7820 100644 --- a/src/app/test/test_manager.cc +++ b/src/app/test/test_manager.cc @@ -1,7 +1,13 @@ #include "app/test/test_manager.h" +#include +#include + #include "absl/strings/str_format.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "app/core/features.h" #include "app/core/platform/file_dialog.h" #include "app/gfx/arena.h" @@ -1281,5 +1287,199 @@ absl::Status TestManager::TestRomDataIntegrity(Rom* rom) { }); } +std::string TestManager::RegisterHarnessTest(const std::string& name, + const std::string& category) { + absl::MutexLock lock(&harness_history_mutex_); + + const std::string sanitized_category = category.empty() ? "grpc" : category; + std::string test_id = GenerateHarnessTestIdLocked(sanitized_category); + + HarnessTestExecution execution; + execution.test_id = test_id; + execution.name = name; + execution.category = sanitized_category; + execution.status = HarnessTestStatus::kQueued; + execution.queued_at = absl::Now(); + execution.started_at = absl::InfinitePast(); + execution.completed_at = absl::InfinitePast(); + + harness_history_[test_id] = execution; + harness_history_order_.push_back(test_id); + TrimHarnessHistoryLocked(); + + HarnessAggregate& aggregate = harness_aggregates_[name]; + if (aggregate.category.empty()) { + aggregate.category = sanitized_category; + } + aggregate.last_run = execution.queued_at; + aggregate.latest_execution = execution; + + return test_id; +} + +void TestManager::MarkHarnessTestRunning(const std::string& test_id) { + absl::MutexLock lock(&harness_history_mutex_); + + auto it = harness_history_.find(test_id); + if (it == harness_history_.end()) { + return; + } + + HarnessTestExecution& execution = it->second; + execution.status = HarnessTestStatus::kRunning; + execution.started_at = absl::Now(); + + HarnessAggregate& aggregate = harness_aggregates_[execution.name]; + if (aggregate.category.empty()) { + aggregate.category = execution.category; + } + aggregate.latest_execution = execution; +} + +void TestManager::MarkHarnessTestCompleted( + const std::string& test_id, HarnessTestStatus status, + const std::string& error_message, + const std::vector& assertion_failures, + const std::vector& logs, + const std::map& metrics) { + absl::MutexLock lock(&harness_history_mutex_); + + auto it = harness_history_.find(test_id); + if (it == harness_history_.end()) { + return; + } + + HarnessTestExecution& execution = it->second; + execution.status = status; + if (execution.started_at == absl::InfinitePast()) { + execution.started_at = execution.queued_at; + } + execution.completed_at = absl::Now(); + execution.duration = execution.completed_at - execution.started_at; + execution.error_message = error_message; + if (!assertion_failures.empty()) { + execution.assertion_failures = assertion_failures; + } + if (!logs.empty()) { + execution.logs.insert(execution.logs.end(), logs.begin(), logs.end()); + } + if (!metrics.empty()) { + execution.metrics.insert(metrics.begin(), metrics.end()); + } + + HarnessAggregate& aggregate = harness_aggregates_[execution.name]; + if (aggregate.category.empty()) { + aggregate.category = execution.category; + } + aggregate.total_runs += 1; + if (status == HarnessTestStatus::kPassed) { + aggregate.pass_count += 1; + } else if (status == HarnessTestStatus::kFailed || + status == HarnessTestStatus::kTimeout) { + aggregate.fail_count += 1; + } + aggregate.total_duration += execution.duration; + aggregate.last_run = execution.completed_at; + aggregate.latest_execution = execution; +} + +void TestManager::AppendHarnessTestLog(const std::string& test_id, + const std::string& log_entry) { + absl::MutexLock lock(&harness_history_mutex_); + + auto it = harness_history_.find(test_id); + if (it == harness_history_.end()) { + return; + } + + HarnessTestExecution& execution = it->second; + execution.logs.push_back(log_entry); + + HarnessAggregate& aggregate = harness_aggregates_[execution.name]; + aggregate.latest_execution.logs = execution.logs; +} + +absl::StatusOr TestManager::GetHarnessTestExecution( + const std::string& test_id) const { + absl::MutexLock lock(&harness_history_mutex_); + + auto it = harness_history_.find(test_id); + if (it == harness_history_.end()) { + return absl::NotFoundError( + absl::StrFormat("Test ID '%s' not found", test_id)); + } + + return it->second; +} + +std::vector TestManager::ListHarnessTestSummaries( + const std::string& category_filter) const { + absl::MutexLock lock(&harness_history_mutex_); + std::vector summaries; + summaries.reserve(harness_aggregates_.size()); + + for (const auto& [name, aggregate] : harness_aggregates_) { + if (!category_filter.empty() && aggregate.category != category_filter) { + continue; + } + + HarnessTestSummary summary; + summary.latest_execution = aggregate.latest_execution; + summary.total_runs = aggregate.total_runs; + summary.pass_count = aggregate.pass_count; + summary.fail_count = aggregate.fail_count; + summary.total_duration = aggregate.total_duration; + summaries.push_back(summary); + } + + std::sort(summaries.begin(), summaries.end(), + [](const HarnessTestSummary& a, const HarnessTestSummary& b) { + absl::Time time_a = a.latest_execution.completed_at; + if (time_a == absl::InfinitePast()) { + time_a = a.latest_execution.queued_at; + } + absl::Time time_b = b.latest_execution.completed_at; + if (time_b == absl::InfinitePast()) { + time_b = b.latest_execution.queued_at; + } + return time_a > time_b; + }); + + return summaries; +} + +std::string TestManager::GenerateHarnessTestIdLocked(absl::string_view prefix) { + static std::mt19937 rng(std::random_device{}()); + static std::uniform_int_distribution dist(0, 0xFFFFFF); + + std::string sanitized = absl::StrReplaceAll(std::string(prefix), + {{" ", "_"}, {":", "_"}}); + if (sanitized.empty()) { + sanitized = "test"; + } + + for (int attempt = 0; attempt < 8; ++attempt) { + std::string candidate = + absl::StrFormat("%s_%08x", sanitized, dist(rng)); + if (harness_history_.find(candidate) == harness_history_.end()) { + return candidate; + } + } + + return absl::StrFormat("%s_%lld", sanitized, + static_cast(absl::ToUnixMillis(absl::Now()))); +} + +void TestManager::TrimHarnessHistoryLocked() { + while (harness_history_order_.size() > harness_history_limit_) { + const std::string& oldest_id = harness_history_order_.front(); + auto it = harness_history_.find(oldest_id); + if (it != harness_history_.end()) { + harness_history_.erase(it); + } + harness_history_order_.pop_front(); + } +} + } // namespace test } // namespace yaze diff --git a/src/app/test/test_manager.h b/src/app/test/test_manager.h index cf1ea163..efeaaec5 100644 --- a/src/app/test/test_manager.h +++ b/src/app/test/test_manager.h @@ -2,13 +2,19 @@ #define YAZE_APP_TEST_TEST_MANAGER_H #include +#include #include +#include #include #include #include #include #include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/synchronization/mutex.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" #include "app/rom.h" #include "imgui.h" #include "util/log.h" @@ -111,6 +117,39 @@ struct ResourceStats { std::chrono::time_point timestamp; }; +// Test harness execution tracking for gRPC automation (IT-05) +enum class HarnessTestStatus { + kUnspecified, + kQueued, + kRunning, + kPassed, + kFailed, + kTimeout, +}; + +struct HarnessTestExecution { + std::string test_id; + std::string name; + std::string category; + HarnessTestStatus status = HarnessTestStatus::kUnspecified; + absl::Time queued_at; + absl::Time started_at; + absl::Time completed_at; + absl::Duration duration = absl::ZeroDuration(); + std::string error_message; + std::vector assertion_failures; + std::vector logs; + std::map metrics; +}; + +struct HarnessTestSummary { + HarnessTestExecution latest_execution; + int total_runs = 0; + int pass_count = 0; + int fail_count = 0; + absl::Duration total_duration = absl::ZeroDuration(); +}; + // Main test manager - singleton class TestManager { public: @@ -209,6 +248,29 @@ class TestManager { } // File dialog mode now uses global feature flags + // Harness test introspection (IT-05) + std::string RegisterHarnessTest(const std::string& name, + const std::string& category) + ABSL_LOCKS_EXCLUDED(harness_history_mutex_); + void MarkHarnessTestRunning(const std::string& test_id) + ABSL_LOCKS_EXCLUDED(harness_history_mutex_); + void MarkHarnessTestCompleted( + const std::string& test_id, HarnessTestStatus status, + const std::string& error_message = "", + const std::vector& assertion_failures = {}, + const std::vector& logs = {}, + const std::map& metrics = {}) + ABSL_LOCKS_EXCLUDED(harness_history_mutex_); + void AppendHarnessTestLog(const std::string& test_id, + const std::string& log_entry) + ABSL_LOCKS_EXCLUDED(harness_history_mutex_); + absl::StatusOr GetHarnessTestExecution( + const std::string& test_id) const + ABSL_LOCKS_EXCLUDED(harness_history_mutex_); + std::vector ListHarnessTestSummaries( + const std::string& category_filter = "") const + ABSL_LOCKS_EXCLUDED(harness_history_mutex_); + private: TestManager(); ~TestManager(); @@ -263,6 +325,31 @@ class TestManager { // Test selection and configuration std::unordered_map disabled_tests_; + + // Harness test tracking + struct HarnessAggregate { + int total_runs = 0; + int pass_count = 0; + int fail_count = 0; + absl::Duration total_duration = absl::ZeroDuration(); + std::string category; + absl::Time last_run; + HarnessTestExecution latest_execution; + }; + + std::unordered_map harness_history_ + ABSL_GUARDED_BY(harness_history_mutex_); + std::unordered_map harness_aggregates_ + ABSL_GUARDED_BY(harness_history_mutex_); + std::deque harness_history_order_ + ABSL_GUARDED_BY(harness_history_mutex_); + size_t harness_history_limit_ = 200; + mutable absl::Mutex harness_history_mutex_; + + std::string GenerateHarnessTestIdLocked(absl::string_view prefix) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(harness_history_mutex_); + void TrimHarnessHistoryLocked() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(harness_history_mutex_); }; // Utility functions for test result formatting