diff --git a/docs/z3ed/E6-z3ed-reference.md b/docs/z3ed/E6-z3ed-reference.md index b3285170..fc5a1a17 100644 --- a/docs/z3ed/E6-z3ed-reference.md +++ b/docs/z3ed/E6-z3ed-reference.md @@ -316,6 +316,29 @@ assertions_passed: 3 assertions_failed: 0 ``` +**Usage Example**: +```bash +$ z3ed agent test status --test-id grpc_wait_20251002T182455 --follow +=== Test Status === +Test ID: grpc_wait_20251002T182455 +Server: localhost:50052 +Follow mode: polling every 1000ms + +Status: RUNNING +Queued At: 2025-10-02T18:24:55Z +Started At: 2025-10-02T18:24:55Z +Completed At: n/a +Execution Time (ms): 432 +Assertion Failures: 0 +--- +Status: PASSED +Queued At: 2025-10-02T18:24:55Z +Started At: 2025-10-02T18:24:55Z +Completed At: 2025-10-02T18:24:55Z +Execution Time (ms): 612 +Assertion Failures: 0 +``` + ##### `agent test results` - Get detailed test results ```bash z3ed agent test results --test-id [--format ] [--include-logs] @@ -329,6 +352,45 @@ Example: z3ed agent test results --test-id grpc_click_12345678 --include-logs ``` +**Usage Example (YAML)**: +```bash +$ z3ed agent test results --test-id grpc_assert_20251002T182500 --include-logs +test_id: grpc_assert_20251002T182500 +success: true +name: "grpc assert Overworld" +category: "grpc" +executed_at: 2025-10-02T18:25:00Z +duration_ms: 118 +assertions: + - description: "Overworld window visible" + passed: true +logs: + - "[2025-10-02T18:25:00Z] Queued assertion" + - "[2025-10-02T18:25:00Z] Assertion passed" +metrics: + execution_frames: 2 +``` + +**Usage Example (JSON)**: +```bash +$ z3ed agent test results --test-id grpc_assert_20251002T182500 --format json +{ + "test_id": "grpc_assert_20251002T182500", + "success": true, + "name": "grpc assert Overworld", + "category": "grpc", + "executed_at": "2025-10-02T18:25:00Z", + "duration_ms": 118, + "assertions": [ + {"description": "Overworld window visible", "passed": true} + ], + "logs": [], + "metrics": { + "execution_frames": 2 + } +} +``` + ##### `agent test list` - List all tests ```bash z3ed agent test list [--category ] [--status ] @@ -341,6 +403,30 @@ Example: z3ed agent test list --category grpc --status failed ``` +**Usage Example**: +```bash +$ z3ed agent test list --category grpc --limit 3 +=== Harness Test Catalog === +Server: localhost:50052 +Category filter: grpc + +Test ID: grpc_click_20251002T182440 + Name: grpc click Open Overworld + Category: grpc + Last Run: 2025-10-02T18:24:41Z + Runs: 5 (5 pass / 0 fail) + Average Duration (ms): 327 + +Test ID: grpc_wait_20251002T182455 + Name: grpc wait Overworld visible + Category: grpc + Last Run: 2025-10-02T18:24:55Z + Runs: 5 (5 pass / 0 fail) + Average Duration (ms): 614 + +Displayed 2 test(s) (catalog size: 42). +``` + #### `agent test record` - Record test sessions (IT-07) ##### `agent test record start` - Begin recording diff --git a/scripts/test_introspection_e2e.sh b/scripts/test_introspection_e2e.sh new file mode 100755 index 00000000..44953276 --- /dev/null +++ b/scripts/test_introspection_e2e.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# End-to-end smoke test for test introspection CLI commands +# Requires YAZE to be built with gRPC support (build-grpc-test preset) + +set -euo pipefail + +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +Z3ED_BIN="./build-grpc-test/bin/z3ed" +YAZE_BIN="./build-grpc-test/bin/yaze.app/Contents/MacOS/yaze" +ROM_FILE="assets/zelda3.sfc" +TEST_PORT="${TEST_PORT:-50052}" +PROMPT="Open Overworld editor and verify it loads" +HOST="localhost" + +STATUS_LOG="$(mktemp /tmp/z3ed_status_XXXX.log)" +RESULTS_LOG="$(mktemp /tmp/z3ed_results_XXXX.log)" +LIST_LOG="$(mktemp /tmp/z3ed_list_XXXX.log)" +RUN_LOG="$(mktemp /tmp/z3ed_run_XXXX.log)" + +cleanup() { + if [[ -n "${YAZE_PID:-}" ]]; then + kill "${YAZE_PID}" 2>/dev/null || true + fi + rm -f "$STATUS_LOG" "$RESULTS_LOG" "$LIST_LOG" "$RUN_LOG" +} +trap cleanup EXIT + +if [[ ! -x "$Z3ED_BIN" ]]; then + echo -e "${RED}Error:${NC} z3ed binary not found at $Z3ED_BIN" + echo "Build with: cmake --build build-grpc-test --target z3ed" + exit 1 +fi + +if [[ ! -x "$YAZE_BIN" ]]; then + echo -e "${RED}Error:${NC} YAZE binary not found at $YAZE_BIN" + echo "Build with: cmake --build build-grpc-test --target yaze" + exit 1 +fi + +if [[ ! -f "$ROM_FILE" ]]; then + echo -e "${RED}Error:${NC} ROM file not found at $ROM_FILE" + exit 1 +fi + +echo -e "${YELLOW}=== Test Harness Introspection E2E ===${NC}" + +# Ensure no previous YAZE instance is running +killall yaze 2>/dev/null || true +sleep 1 + +echo -e "${BLUE}→ Starting YAZE (port $TEST_PORT)...${NC}" +"$YAZE_BIN" \ + --enable_test_harness \ + --test_harness_port="$TEST_PORT" \ + --rom_file="$ROM_FILE" & +YAZE_PID=$! + +ready=0 +for attempt in {1..20}; do + if lsof -i ":$TEST_PORT" >/dev/null 2>&1; then + ready=1 + break + fi + sleep 0.5 +done + +if [[ "$ready" -ne 1 ]]; then + echo -e "${RED}Error:${NC} ImGuiTestHarness server did not start on port $TEST_PORT" + exit 1 +fi + +echo -e "${GREEN}✓ Harness ready${NC}" + +echo -e "${BLUE}→ Running agent test workflow: $PROMPT${NC}" +if ! "$Z3ED_BIN" agent test --prompt "$PROMPT" --host "$HOST" --port "$TEST_PORT" | tee "$RUN_LOG"; then + echo -e "${RED}Error:${NC} agent test run failed" + exit 1 +fi + +PRIMARY_TEST_ID=$(sed -n 's/.*Test ID: \([^][]*\).*/\1/p' "$RUN_LOG" | tail -n 1 | tr -d ' ]') +if [[ -z "$PRIMARY_TEST_ID" ]]; then + echo -e "${RED}Error:${NC} Unable to extract test id from agent test output" + exit 1 +fi + +echo -e "${GREEN}✓ Captured Test ID:${NC} $PRIMARY_TEST_ID" + +echo -e "${BLUE}→ Checking status${NC}" +"$Z3ED_BIN" agent test status --test-id "$PRIMARY_TEST_ID" --host "$HOST" --port "$TEST_PORT" | tee "$STATUS_LOG" +if ! grep -q "Status: " "$STATUS_LOG"; then + echo -e "${RED}Error:${NC} status command did not return a status" + exit 1 +fi + +if grep -q "Status: PASSED" "$STATUS_LOG"; then + echo -e "${GREEN}✓ Status indicates PASS${NC}" +else + echo -e "${YELLOW}! Status is not PASSED (see $STATUS_LOG)${NC}" +fi + +echo -e "${BLUE}→ Fetching detailed results (YAML)${NC}" +"$Z3ED_BIN" agent test results --test-id "$PRIMARY_TEST_ID" --include-logs --host "$HOST" --port "$TEST_PORT" | tee "$RESULTS_LOG" +if ! grep -q "success: " "$RESULTS_LOG"; then + echo -e "${RED}Error:${NC} results command failed" + exit 1 +fi + +echo -e "${BLUE}→ Listing recent grpc tests${NC}" +"$Z3ED_BIN" agent test list --category grpc --limit 5 --host "$HOST" --port "$TEST_PORT" | tee "$LIST_LOG" +if ! grep -q "Test ID:" "$LIST_LOG"; then + echo -e "${RED}Error:${NC} list command returned no tests" + exit 1 +fi + +echo -e "${GREEN}✓ Introspection commands completed successfully${NC}" + +echo -e "${YELLOW}Artifacts:${NC}" +echo " Status log: $STATUS_LOG" +echo " Results log: $RESULTS_LOG" +echo " List log: $LIST_LOG" + +echo -e "${GREEN}All checks passed!${NC}" +exit 0 diff --git a/src/app/core/imgui_test_harness_service.cc b/src/app/core/imgui_test_harness_service.cc index b8711da1..72111afc 100644 --- a/src/app/core/imgui_test_harness_service.cc +++ b/src/app/core/imgui_test_harness_service.cc @@ -44,23 +44,22 @@ void KeepDynamicTestData(const std::shared_ptr& data) { } } -GetTestStatusResponse::Status ConvertHarnessStatus( - TestManager::HarnessTestStatus status) { - using ProtoStatus = GetTestStatusResponse::Status; +::yaze::test::GetTestStatusResponse_Status ConvertHarnessStatus( + ::yaze::test::HarnessTestStatus status) { switch (status) { - case TestManager::HarnessTestStatus::kQueued: - return ProtoStatus::STATUS_QUEUED; - case TestManager::HarnessTestStatus::kRunning: - return ProtoStatus::STATUS_RUNNING; - case TestManager::HarnessTestStatus::kPassed: - return ProtoStatus::STATUS_PASSED; - case TestManager::HarnessTestStatus::kFailed: - return ProtoStatus::STATUS_FAILED; - case TestManager::HarnessTestStatus::kTimeout: - return ProtoStatus::STATUS_TIMEOUT; - case TestManager::HarnessTestStatus::kUnspecified: + case ::yaze::test::HarnessTestStatus::kQueued: + return ::yaze::test::GetTestStatusResponse::STATUS_QUEUED; + case ::yaze::test::HarnessTestStatus::kRunning: + return ::yaze::test::GetTestStatusResponse::STATUS_RUNNING; + case ::yaze::test::HarnessTestStatus::kPassed: + return ::yaze::test::GetTestStatusResponse::STATUS_PASSED; + case ::yaze::test::HarnessTestStatus::kFailed: + return ::yaze::test::GetTestStatusResponse::STATUS_FAILED; + case ::yaze::test::HarnessTestStatus::kTimeout: + return ::yaze::test::GetTestStatusResponse::STATUS_TIMEOUT; + case ::yaze::test::HarnessTestStatus::kUnspecified: default: - return ProtoStatus::STATUS_UNSPECIFIED; + return ::yaze::test::GetTestStatusResponse::STATUS_UNSPECIFIED; } } @@ -296,8 +295,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Click(const ClickRequest* request, response->set_success(false); response->set_message(message); response->set_execution_time_ms(elapsed.count()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); return absl::OkStatus(); } @@ -311,8 +310,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Click(const ClickRequest* request, response->set_success(false); response->set_message(message); response->set_execution_time_ms(elapsed.count()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } @@ -353,16 +352,16 @@ absl::Status ImGuiTestHarnessServiceImpl::Click(const ClickRequest* request, const std::string success_message = absl::StrFormat("Clicked %s '%s'", widget_type, widget_label); manager->AppendHarnessTestLog(captured_id, success_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kPassed, - success_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kPassed, + success_message); } catch (const std::exception& e) { const std::string error_message = absl::StrFormat("Click failed: %s", e.what()); manager->AppendHarnessTestLog(captured_id, error_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kFailed, - error_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kFailed, + error_message); } }; @@ -397,8 +396,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Click(const ClickRequest* request, response->set_success(false); response->set_message(message); response->set_execution_time_ms(elapsed.count()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted(test_id, + HarnessTestStatus::kFailed, + message); test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } @@ -410,8 +410,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Click(const ClickRequest* request, widget_type, widget_label); test_manager_->MarkHarnessTestRunning(test_id); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->MarkHarnessTestCompleted(test_id, + HarnessTestStatus::kPassed, + message); test_manager_->AppendHarnessTestLog(test_id, message); auto elapsed = std::chrono::duration_cast( @@ -450,8 +451,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, response->set_success(false); response->set_message(message); response->set_execution_time_ms(elapsed.count()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); return absl::OkStatus(); } @@ -465,8 +466,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, response->set_success(false); response->set_message(message); response->set_execution_time_ms(elapsed.count()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } @@ -480,7 +481,7 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, auto test_data = std::make_shared(); TestManager* manager = test_manager_; test_data->test_func = [manager, captured_id = test_id, widget_type, - widget_label, clear_first, text]( + widget_label, clear_first, text, rpc_state]( ImGuiTestContext* ctx) { manager->MarkHarnessTestRunning(captured_id); try { @@ -489,9 +490,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, std::string error_message = absl::StrFormat("Input field '%s' not found", widget_label); manager->AppendHarnessTestLog(captured_id, error_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kFailed, - error_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kFailed, + error_message); rpc_state->SetResult(false, error_message); return; } @@ -508,17 +509,17 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, "Typed '%s' into %s '%s'%s", text, widget_type, widget_label, clear_first ? " (cleared first)" : ""); manager->AppendHarnessTestLog(captured_id, success_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kPassed, - success_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kPassed, + success_message); rpc_state->SetResult(true, success_message); } catch (const std::exception& e) { std::string error_message = absl::StrFormat("Type failed: %s", e.what()); manager->AppendHarnessTestLog(captured_id, error_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kFailed, - error_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kFailed, + error_message); rpc_state->SetResult(false, error_message); } }; @@ -542,8 +543,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, std::string error_message = "Test timeout - input field not found or unresponsive"; manager->AppendHarnessTestLog(test_id, error_message); - manager->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kTimeout, error_message); + manager->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kTimeout, error_message); rpc_state->SetResult(false, error_message); break; } @@ -568,8 +569,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Type(const TypeRequest* request, std::string message = absl::StrFormat( "[STUB] Typed '%s' into %s (ImGuiTestEngine not available)", request->text(), request->target()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->MarkHarnessTestCompleted(test_id, + HarnessTestStatus::kPassed, + message); test_manager_->AppendHarnessTestLog(test_id, message); auto elapsed = std::chrono::duration_cast( @@ -609,8 +611,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Wait(const WaitRequest* request, response->set_success(false); response->set_message(message); response->set_elapsed_ms(elapsed.count()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); return absl::OkStatus(); } @@ -624,8 +626,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Wait(const WaitRequest* request, response->set_success(false); response->set_message(message); response->set_elapsed_ms(elapsed.count()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } @@ -672,9 +674,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Wait(const WaitRequest* request, std::string error_message = absl::StrFormat("Unknown condition type: %s", condition_type); manager->AppendHarnessTestLog(captured_id, error_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kFailed, - error_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kFailed, + error_message); return; } @@ -685,9 +687,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Wait(const WaitRequest* request, "Condition '%s:%s' met after %lld ms", condition_type, condition_target, static_cast(elapsed_ms.count())); manager->AppendHarnessTestLog(captured_id, success_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kPassed, - success_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kPassed, + success_message); return; } @@ -700,16 +702,16 @@ absl::Status ImGuiTestHarnessServiceImpl::Wait(const WaitRequest* request, "Condition '%s:%s' not met after %d ms timeout", condition_type, condition_target, timeout_ms); manager->AppendHarnessTestLog(captured_id, timeout_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kTimeout, - timeout_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kTimeout, + timeout_message); } catch (const std::exception& e) { std::string error_message = absl::StrFormat("Wait failed: %s", e.what()); manager->AppendHarnessTestLog(captured_id, error_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kFailed, - error_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kFailed, + error_message); } }; @@ -739,8 +741,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Wait(const WaitRequest* request, std::string message = absl::StrFormat( "[STUB] Condition '%s' met (ImGuiTestEngine not available)", request->condition()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->MarkHarnessTestCompleted(test_id, + HarnessTestStatus::kPassed, + message); test_manager_->AppendHarnessTestLog(test_id, message); auto elapsed = std::chrono::duration_cast( @@ -777,8 +780,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request, response->set_message(message); response->set_actual_value("N/A"); response->set_expected_value("N/A"); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); return absl::OkStatus(); } @@ -791,8 +794,8 @@ absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request, response->set_message(message); response->set_actual_value("N/A"); response->set_expected_value("N/A"); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kFailed, message); + test_manager_->MarkHarnessTestCompleted( + test_id, HarnessTestStatus::kFailed, message); test_manager_->AppendHarnessTestLog(test_id, message); return absl::OkStatus(); } @@ -806,11 +809,11 @@ absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request, assertion_target](ImGuiTestContext* ctx) { manager->MarkHarnessTestRunning(captured_id); - auto complete_with = - [manager, captured_id](bool passed, const std::string& message, - const std::string& actual, - const std::string& expected, - TestManager::HarnessTestStatus status) { + auto complete_with = + [manager, captured_id](bool passed, const std::string& message, + const std::string& actual, + const std::string& expected, + HarnessTestStatus status) { manager->AppendHarnessTestLog(captured_id, message); if (!actual.empty() || !expected.empty()) { manager->AppendHarnessTestLog( @@ -866,7 +869,7 @@ absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request, std::string error_message = "text_contains requires format 'text_contains:target:expected_text'"; complete_with(false, error_message, "N/A", "N/A", - TestManager::HarnessTestStatus::kFailed); + HarnessTestStatus::kFailed); return; } @@ -894,21 +897,21 @@ absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request, } else { std::string error_message = absl::StrFormat("Unknown assertion type: %s", assertion_type); - complete_with(false, error_message, "N/A", "N/A", - TestManager::HarnessTestStatus::kFailed); + complete_with(false, error_message, "N/A", "N/A", + HarnessTestStatus::kFailed); return; } complete_with(passed, message, actual_value, expected_value, - passed ? TestManager::HarnessTestStatus::kPassed - : TestManager::HarnessTestStatus::kFailed); + passed ? HarnessTestStatus::kPassed + : HarnessTestStatus::kFailed); } catch (const std::exception& e) { std::string error_message = absl::StrFormat("Assertion failed: %s", e.what()); manager->AppendHarnessTestLog(captured_id, error_message); - manager->MarkHarnessTestCompleted( - captured_id, TestManager::HarnessTestStatus::kFailed, - error_message); + manager->MarkHarnessTestCompleted(captured_id, + HarnessTestStatus::kFailed, + error_message); } }; @@ -936,8 +939,9 @@ absl::Status ImGuiTestHarnessServiceImpl::Assert(const AssertRequest* request, std::string message = absl::StrFormat( "[STUB] Assertion '%s' passed (ImGuiTestEngine not available)", request->condition()); - test_manager_->MarkHarnessTestCompleted( - test_id, TestManager::HarnessTestStatus::kPassed, message); + test_manager_->MarkHarnessTestCompleted(test_id, + HarnessTestStatus::kPassed, + message); test_manager_->AppendHarnessTestLog(test_id, message); response->set_success(true); @@ -1099,7 +1103,7 @@ absl::Status ImGuiTestHarnessServiceImpl::GetTestResults( const auto& execution = execution_or.value(); response->set_success( - execution.status == TestManager::HarnessTestStatus::kPassed); + execution.status == HarnessTestStatus::kPassed); response->set_test_name(execution.name); response->set_category(execution.category); diff --git a/src/cli/handlers/agent.cc b/src/cli/handlers/agent.cc index bd9fcb30..00756348 100644 --- a/src/cli/handlers/agent.cc +++ b/src/cli/handlers/agent.cc @@ -18,11 +18,15 @@ #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" #include "absl/time/time.h" #include // For EXIT_FAILURE #include #include +#include + +#include // Declare the rom flag so we can access it ABSL_DECLARE_FLAG(std::string, rom); @@ -53,6 +57,103 @@ struct DescribeOptions { std::optional last_updated; }; +std::string FormatOptionalTime(const std::optional& time) { + if (!time.has_value()) { + return "n/a"; + } + return absl::FormatTime("%Y-%m-%dT%H:%M:%SZ", *time, absl::UTCTimeZone()); +} + +std::string TestRunStatusToString(TestRunStatus status) { + switch (status) { + case TestRunStatus::kQueued: + return "QUEUED"; + case TestRunStatus::kRunning: + return "RUNNING"; + case TestRunStatus::kPassed: + return "PASSED"; + case TestRunStatus::kFailed: + return "FAILED"; + case TestRunStatus::kTimeout: + return "TIMEOUT"; + case TestRunStatus::kUnknown: + default: + return "UNKNOWN"; + } +} + +bool IsTerminalStatus(TestRunStatus status) { + switch (status) { + case TestRunStatus::kQueued: + case TestRunStatus::kRunning: + return false; + case TestRunStatus::kPassed: + case TestRunStatus::kFailed: + case TestRunStatus::kTimeout: + case TestRunStatus::kUnknown: + default: + return true; + } +} + +std::optional ParseStatusFilter(absl::string_view value) { + std::string lower = std::string(absl::AsciiStrToLower(value)); + if (lower == "queued") return TestRunStatus::kQueued; + if (lower == "running") return TestRunStatus::kRunning; + if (lower == "passed") return TestRunStatus::kPassed; + if (lower == "failed") return TestRunStatus::kFailed; + if (lower == "timeout") return TestRunStatus::kTimeout; + if (lower == "unknown") return TestRunStatus::kUnknown; + return std::nullopt; +} + +std::string HarnessAddress(const std::string& host, int port) { + return absl::StrFormat("%s:%d", host, port); +} + +std::string JsonEscape(absl::string_view value) { + std::string out; + out.reserve(value.size() + 8); + for (unsigned char c : value) { + switch (c) { + case '\\': + out += "\\\\"; + break; + case '"': + out += "\\\""; + break; + case '\b': + out += "\\b"; + break; + case '\f': + out += "\\f"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + if (c < 0x20) { + absl::StrAppend(&out, absl::StrFormat("\\\\u%04X", static_cast(c))); + } else { + out.push_back(static_cast(c)); + } + } + } + return out; +} + +std::string YamlQuote(absl::string_view value) { + std::string escaped(value); + absl::StrReplaceAll({{"\\", "\\\\"}, {"\"", "\\\""}}, &escaped); + return absl::StrCat("\"", escaped, "\""); +} + absl::StatusOr ParseDescribeArgs( const std::vector& args) { DescribeOptions options; @@ -355,7 +456,7 @@ absl::Status HandleDiffCommand(Rom& rom, const std::vector& args) { return absl::OkStatus(); } -absl::Status HandleTestCommand(const std::vector& arg_vec) { +absl::Status HandleTestRunCommand(const std::vector& arg_vec) { // Parse arguments std::string prompt; std::string host = "localhost"; @@ -413,7 +514,7 @@ absl::Status HandleTestCommand(const std::vector& arg_vec) { std::cout << "Generated workflow:\n" << workflow.ToString() << "\n"; // Connect to test harness - GuiAutomationClient client(absl::StrFormat("%s:%d", host, port)); + GuiAutomationClient client(HarnessAddress(host, port)); auto connect_status = client.Connect(); if (!connect_status.ok()) { return absl::UnavailableError( @@ -430,6 +531,7 @@ absl::Status HandleTestCommand(const std::vector& arg_vec) { // Execute workflow auto start_time = std::chrono::steady_clock::now(); int step_num = 0; + std::vector emitted_test_ids; for (const auto& step : workflow.steps) { step_num++; @@ -471,8 +573,13 @@ absl::Status HandleTestCommand(const std::vector& arg_vec) { absl::StrFormat("Step %d failed: %s", step_num, result->message)); } - std::cout << absl::StrFormat("✓ (%lldms)\n", + std::cout << absl::StrFormat("✓ (%lldms)", result->execution_time.count()); + if (!result->test_id.empty()) { + std::cout << " [Test ID: " << result->test_id << "]"; + emitted_test_ids.push_back(result->test_id); + } + std::cout << "\n"; } auto end_time = std::chrono::steady_clock::now(); @@ -480,10 +587,472 @@ absl::Status HandleTestCommand(const std::vector& arg_vec) { end_time - start_time); std::cout << "\n✅ Test passed in " << elapsed.count() << "ms\n"; + + if (!emitted_test_ids.empty()) { + std::cout << "Latest Test ID: " << emitted_test_ids.back() << "\n"; + if (emitted_test_ids.size() > 1) { + std::cout << "Captured Test IDs:\n"; + for (const auto& id : emitted_test_ids) { + std::cout << " - " << id << "\n"; + } + } + std::cout << "Use 'z3ed agent test status --test-id " << emitted_test_ids.back() + << "' for live status updates." << std::endl; + } + return absl::OkStatus(); #endif } +absl::Status HandleTestStatusCommand(const std::vector& arg_vec) { + std::string host = "localhost"; + int port = 50052; + std::string test_id; + bool follow = false; + int interval_ms = 1000; + + for (size_t i = 0; i < arg_vec.size(); ++i) { + const std::string& token = arg_vec[i]; + + if (token == "--test-id" && i + 1 < arg_vec.size()) { + test_id = arg_vec[++i]; + } else if (absl::StartsWith(token, "--test-id=")) { + test_id = token.substr(10); + } else if (token == "--host" && i + 1 < arg_vec.size()) { + host = arg_vec[++i]; + } else if (absl::StartsWith(token, "--host=")) { + host = token.substr(7); + } else if (token == "--port" && i + 1 < arg_vec.size()) { + port = std::stoi(arg_vec[++i]); + } else if (absl::StartsWith(token, "--port=")) { + port = std::stoi(token.substr(7)); + } else if (token == "--follow") { + follow = true; + } else if ((token == "--interval" || token == "--interval-ms") && + i + 1 < arg_vec.size()) { + interval_ms = std::max(100, std::stoi(arg_vec[++i])); + } else if (absl::StartsWith(token, "--interval=") || + absl::StartsWith(token, "--interval-ms=")) { + size_t prefix = token.find('='); + interval_ms = std::max(100, std::stoi(token.substr(prefix + 1))); + } + } + + if (test_id.empty()) { + return absl::InvalidArgumentError( + "Usage: agent test status --test-id [--follow] [--host ] [--port ] [--interval-ms ]"); + } + +#ifndef YAZE_WITH_GRPC + return absl::UnimplementedError( + "GUI automation requires YAZE_WITH_GRPC=ON at build time.\n" + "Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON"); +#else + GuiAutomationClient client(HarnessAddress(host, port)); + RETURN_IF_ERROR(client.Connect()); + + std::cout << "\n=== Test Status ===\n"; + std::cout << "Test ID: " << test_id << "\n"; + std::cout << "Server: " << HarnessAddress(host, port) << "\n"; + if (follow) { + std::cout << "Follow mode: polling every " << interval_ms << "ms\n"; + } + std::cout << "\n"; + + bool first_iteration = true; + while (true) { + ASSIGN_OR_RETURN(auto details, client.GetTestStatus(test_id)); + + if (!first_iteration) { + std::cout << "---\n"; + } + + std::cout << "Status: " << TestRunStatusToString(details.status) << "\n"; + std::cout << "Queued At: " << FormatOptionalTime(details.queued_at) << "\n"; + std::cout << "Started At: " << FormatOptionalTime(details.started_at) << "\n"; + std::cout << "Completed At: " << FormatOptionalTime(details.completed_at) << "\n"; + std::cout << "Execution Time (ms): " << details.execution_time_ms << "\n"; + if (!details.error_message.empty()) { + std::cout << "Error: " << details.error_message << "\n"; + } + + if (!details.assertion_failures.empty()) { + std::cout << "Assertion Failures (" << details.assertion_failures.size() + << "):\n"; + for (const auto& failure : details.assertion_failures) { + std::cout << " - " << failure << "\n"; + } + } else { + std::cout << "Assertion Failures: 0\n"; + } + + if (!follow || IsTerminalStatus(details.status)) { + break; + } + + first_iteration = false; + std::this_thread::sleep_for(std::chrono::milliseconds(interval_ms)); + } + + return absl::OkStatus(); +#endif +} + +absl::Status HandleTestListCommand(const std::vector& arg_vec) { + std::string host = "localhost"; + int port = 50052; + std::string category_filter; + std::optional status_filter; + int page_size = 100; + int limit = -1; + bool fetch_all = false; + + for (size_t i = 0; i < arg_vec.size(); ++i) { + const std::string& token = arg_vec[i]; + + if (token == "--host" && i + 1 < arg_vec.size()) { + host = arg_vec[++i]; + } else if (absl::StartsWith(token, "--host=")) { + host = token.substr(7); + } else if (token == "--port" && i + 1 < arg_vec.size()) { + port = std::stoi(arg_vec[++i]); + } else if (absl::StartsWith(token, "--port=")) { + port = std::stoi(token.substr(7)); + } else if (token == "--category" && i + 1 < arg_vec.size()) { + category_filter = arg_vec[++i]; + } else if (absl::StartsWith(token, "--category=")) { + category_filter = token.substr(11); + } else if (token == "--status" && i + 1 < arg_vec.size()) { + auto parsed = ParseStatusFilter(arg_vec[++i]); + if (!parsed.has_value()) { + return absl::InvalidArgumentError( + "Invalid status filter. Expected: queued, running, passed, failed, timeout, unknown"); + } + status_filter = parsed; + } else if (absl::StartsWith(token, "--status=")) { + auto parsed = ParseStatusFilter(token.substr(9)); + if (!parsed.has_value()) { + return absl::InvalidArgumentError( + "Invalid status filter. Expected: queued, running, passed, failed, timeout, unknown"); + } + status_filter = parsed; + } else if (token == "--page-size" && i + 1 < arg_vec.size()) { + page_size = std::max(1, std::stoi(arg_vec[++i])); + } else if (absl::StartsWith(token, "--page-size=")) { + page_size = std::max(1, std::stoi(token.substr(12))); + } else if (token == "--limit" && i + 1 < arg_vec.size()) { + limit = std::stoi(arg_vec[++i]); + } else if (absl::StartsWith(token, "--limit=")) { + limit = std::stoi(token.substr(8)); + } else if (token == "--all") { + fetch_all = true; + } + } + + if (fetch_all) { + limit = -1; + } + +#ifndef YAZE_WITH_GRPC + return absl::UnimplementedError( + "GUI automation requires YAZE_WITH_GRPC=ON at build time.\n" + "Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON"); +#else + GuiAutomationClient client(HarnessAddress(host, port)); + RETURN_IF_ERROR(client.Connect()); + + std::cout << "\n=== Harness Test Catalog ===\n"; + std::cout << "Server: " << HarnessAddress(host, port) << "\n"; + if (!category_filter.empty()) { + std::cout << "Category filter: " << category_filter << "\n"; + } + if (status_filter.has_value()) { + std::cout << "Status filter: " + << TestRunStatusToString(status_filter.value()) << "\n"; + } + std::cout << "\n"; + + std::vector collected; + collected.reserve(limit > 0 ? limit : page_size); + std::string page_token; + int total_count = 0; + + while (true) { + int request_page_size = page_size > 0 ? page_size : 100; + if (limit > 0) { + int remaining = limit - static_cast(collected.size()); + if (remaining <= 0) { + break; + } + request_page_size = std::min(request_page_size, remaining); + } + + ASSIGN_OR_RETURN(auto batch, + client.ListTests(category_filter, request_page_size, + page_token)); + + total_count = batch.total_count; + + for (const auto& summary : batch.tests) { + if (status_filter.has_value()) { + ASSIGN_OR_RETURN(auto details, + client.GetTestStatus(summary.test_id)); + if (details.status != status_filter.value()) { + continue; + } + } + + collected.push_back(summary); + if (limit > 0 && static_cast(collected.size()) >= limit) { + break; + } + } + + if (limit > 0 && static_cast(collected.size()) >= limit) { + break; + } + + if (batch.next_page_token.empty()) { + break; + } + page_token = batch.next_page_token; + } + + if (collected.empty()) { + std::cout << "No tests found for the specified filters." << std::endl; + return absl::OkStatus(); + } + + for (const auto& summary : collected) { + std::cout << "Test ID: " << summary.test_id << "\n"; + std::cout << " Name: " << summary.name << "\n"; + std::cout << " Category: " << summary.category << "\n"; + std::cout << " Last Run: " << FormatOptionalTime(summary.last_run_at) + << "\n"; + std::cout << " Runs: " << summary.total_runs << " (" + << summary.pass_count << " pass / " << summary.fail_count + << " fail)\n"; + std::cout << " Average Duration (ms): " << summary.average_duration_ms + << "\n\n"; + } + + std::cout << "Displayed " << collected.size() << " test(s)"; + if (total_count > 0) { + std::cout << " (catalog size: " << total_count << ")"; + } + std::cout << "." << std::endl; + + return absl::OkStatus(); +#endif +} + +absl::Status HandleTestResultsCommand(const std::vector& arg_vec) { + std::string host = "localhost"; + int port = 50052; + std::string test_id; + bool include_logs = false; + std::string format = "yaml"; + + for (size_t i = 0; i < arg_vec.size(); ++i) { + const std::string& token = arg_vec[i]; + + if (token == "--test-id" && i + 1 < arg_vec.size()) { + test_id = arg_vec[++i]; + } else if (absl::StartsWith(token, "--test-id=")) { + test_id = token.substr(10); + } else if (token == "--host" && i + 1 < arg_vec.size()) { + host = arg_vec[++i]; + } else if (absl::StartsWith(token, "--host=")) { + host = token.substr(7); + } else if (token == "--port" && i + 1 < arg_vec.size()) { + port = std::stoi(arg_vec[++i]); + } else if (absl::StartsWith(token, "--port=")) { + port = std::stoi(token.substr(7)); + } else if (token == "--include-logs") { + include_logs = true; + } else if (token == "--format" && i + 1 < arg_vec.size()) { + format = absl::AsciiStrToLower(arg_vec[++i]); + } else if (absl::StartsWith(token, "--format=")) { + format = absl::AsciiStrToLower(token.substr(9)); + } + } + + if (test_id.empty()) { + return absl::InvalidArgumentError( + "Usage: agent test results --test-id [--include-logs] [--format yaml|json] [--host ] [--port ]"); + } + + if (format != "yaml" && format != "json") { + return absl::InvalidArgumentError("--format must be either 'yaml' or 'json'"); + } + +#ifndef YAZE_WITH_GRPC + return absl::UnimplementedError( + "GUI automation requires YAZE_WITH_GRPC=ON at build time.\n" + "Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON"); +#else + GuiAutomationClient client(HarnessAddress(host, port)); + RETURN_IF_ERROR(client.Connect()); + + ASSIGN_OR_RETURN(auto details, + client.GetTestResults(test_id, include_logs)); + + if (format == "json") { + std::cout << "{\n"; + std::cout << " \"test_id\": \"" << JsonEscape(details.test_id) + << "\",\n"; + std::cout << " \"success\": " << (details.success ? "true" : "false") + << ",\n"; + std::cout << " \"name\": \"" << JsonEscape(details.test_name) + << "\",\n"; + std::cout << " \"category\": \"" << JsonEscape(details.category) + << "\",\n"; + std::cout << " \"executed_at\": \"" + << JsonEscape(FormatOptionalTime(details.executed_at)) + << "\",\n"; + std::cout << " \"duration_ms\": " << details.duration_ms << ",\n"; + + std::cout << " \"assertions\": "; + if (details.assertions.empty()) { + std::cout << "[],\n"; + } else { + std::cout << "[\n"; + for (size_t i = 0; i < details.assertions.size(); ++i) { + const auto& assertion = details.assertions[i]; + std::cout << " {\"description\": \"" + << JsonEscape(assertion.description) + << "\", \"passed\": " + << (assertion.passed ? "true" : "false"); + if (!assertion.expected_value.empty()) { + std::cout << ", \"expected\": \"" + << JsonEscape(assertion.expected_value) << "\""; + } + if (!assertion.actual_value.empty()) { + std::cout << ", \"actual\": \"" + << JsonEscape(assertion.actual_value) << "\""; + } + if (!assertion.error_message.empty()) { + std::cout << ", \"error\": \"" + << JsonEscape(assertion.error_message) << "\""; + } + std::cout << "}"; + if (i + 1 < details.assertions.size()) { + std::cout << ","; + } + std::cout << "\n"; + } + std::cout << " ],\n"; + } + + std::cout << " \"logs\": "; + if (include_logs && !details.logs.empty()) { + std::cout << "[\n"; + for (size_t i = 0; i < details.logs.size(); ++i) { + std::cout << " \"" << JsonEscape(details.logs[i]) << "\""; + if (i + 1 < details.logs.size()) { + std::cout << ","; + } + std::cout << "\n"; + } + std::cout << " ],\n"; + } else { + std::cout << "[],\n"; + } + + std::cout << " \"metrics\": "; + if (!details.metrics.empty()) { + std::cout << "{\n"; + size_t index = 0; + for (const auto& [key, value] : details.metrics) { + std::cout << " \"" << JsonEscape(key) << "\": " << value; + if (index + 1 < details.metrics.size()) { + std::cout << ","; + } + std::cout << "\n"; + ++index; + } + std::cout << " }\n"; + } else { + std::cout << "{}\n"; + } + + std::cout << "}" << std::endl; + } else { + std::cout << "test_id: " << details.test_id << "\n"; + std::cout << "success: " << (details.success ? "true" : "false") + << "\n"; + std::cout << "name: " << YamlQuote(details.test_name) << "\n"; + std::cout << "category: " << YamlQuote(details.category) << "\n"; + std::cout << "executed_at: " << FormatOptionalTime(details.executed_at) + << "\n"; + std::cout << "duration_ms: " << details.duration_ms << "\n"; + + if (details.assertions.empty()) { + std::cout << "assertions: []\n"; + } else { + std::cout << "assertions:\n"; + for (const auto& assertion : details.assertions) { + std::cout << " - description: " + << YamlQuote(assertion.description) << "\n"; + std::cout << " passed: " + << (assertion.passed ? "true" : "false") << "\n"; + if (!assertion.expected_value.empty()) { + std::cout << " expected: " + << YamlQuote(assertion.expected_value) << "\n"; + } + if (!assertion.actual_value.empty()) { + std::cout << " actual: " + << YamlQuote(assertion.actual_value) << "\n"; + } + if (!assertion.error_message.empty()) { + std::cout << " error: " + << YamlQuote(assertion.error_message) << "\n"; + } + } + } + + if (include_logs && !details.logs.empty()) { + std::cout << "logs:\n"; + for (const auto& log : details.logs) { + std::cout << " - " << YamlQuote(log) << "\n"; + } + } else { + std::cout << "logs: []\n"; + } + + if (details.metrics.empty()) { + std::cout << "metrics: {}\n"; + } else { + std::cout << "metrics:\n"; + for (const auto& [key, value] : details.metrics) { + std::cout << " " << key << ": " << value << "\n"; + } + } + } + + return absl::OkStatus(); +#endif +} + +absl::Status HandleTestCommand(const std::vector& arg_vec) { + if (!arg_vec.empty()) { + const std::string& subcommand = arg_vec[0]; + std::vector tail(arg_vec.begin() + 1, arg_vec.end()); + + if (subcommand == "status") { + return HandleTestStatusCommand(tail); + } + if (subcommand == "list") { + return HandleTestListCommand(tail); + } + if (subcommand == "results") { + return HandleTestResultsCommand(tail); + } + } + + return HandleTestRunCommand(arg_vec); +} + absl::Status HandleLearnCommand() { std::cout << "Agent learn not yet implemented." << std::endl; return absl::OkStatus(); diff --git a/src/cli/modern_cli.cc b/src/cli/modern_cli.cc index 2d69ad89..3ba00171 100644 --- a/src/cli/modern_cli.cc +++ b/src/cli/modern_cli.cc @@ -60,7 +60,11 @@ void ModernCLI::SetupCommands() { commands_["agent"] = { .name = "agent", .description = "Interact with the AI agent", - .usage = "z3ed agent [options]\n" + .usage = "z3ed agent [options]\n" + " test run: --prompt \"\" [--host ] [--port ] [--timeout ]\n" + " test status: status --test-id [--follow] [--host ] [--port ]\n" + " test list: list [--category ] [--status ] [--limit ] [--host ] [--port ]\n" + " test results: results --test-id [--include-logs] [--format yaml|json] [--host ] [--port ]\n" " describe options: [--resource ] [--format json|yaml] [--output ]\n" " [--version ] [--last-updated ]", .handler = [this](const std::vector& args) -> absl::Status { diff --git a/src/cli/service/gui_automation_client.cc b/src/cli/service/gui_automation_client.cc index 6db37de6..1e85c20a 100644 --- a/src/cli/service/gui_automation_client.cc +++ b/src/cli/service/gui_automation_client.cc @@ -4,10 +4,46 @@ #include "cli/service/gui_automation_client.h" #include "absl/strings/str_format.h" +#include "absl/time/time.h" + +#include namespace yaze { namespace cli { +namespace { + +#ifdef YAZE_WITH_GRPC +std::optional OptionalTimeFromMillis(int64_t millis) { + if (millis <= 0) { + return std::nullopt; + } + return absl::FromUnixMillis(millis); +} + +TestRunStatus ConvertStatusProto( + yaze::test::GetTestStatusResponse::Status status) { + using ProtoStatus = yaze::test::GetTestStatusResponse::Status; + switch (status) { + case ProtoStatus::GetTestStatusResponse_Status_STATUS_QUEUED: + return TestRunStatus::kQueued; + case ProtoStatus::GetTestStatusResponse_Status_STATUS_RUNNING: + return TestRunStatus::kRunning; + case ProtoStatus::GetTestStatusResponse_Status_STATUS_PASSED: + return TestRunStatus::kPassed; + case ProtoStatus::GetTestStatusResponse_Status_STATUS_FAILED: + return TestRunStatus::kFailed; + case ProtoStatus::GetTestStatusResponse_Status_STATUS_TIMEOUT: + return TestRunStatus::kTimeout; + case ProtoStatus::GetTestStatusResponse_Status_STATUS_UNSPECIFIED: + default: + return TestRunStatus::kUnknown; + } +} +#endif // YAZE_WITH_GRPC + +} // namespace + GuiAutomationClient::GuiAutomationClient(const std::string& server_address) : server_address_(server_address) {} @@ -66,6 +102,7 @@ absl::StatusOr GuiAutomationClient::Ping( response.yaze_version(), response.timestamp_ms()); result.execution_time = std::chrono::milliseconds(0); + result.test_id.clear(); return result; #else return absl::UnimplementedError("gRPC not available"); @@ -84,16 +121,20 @@ absl::StatusOr GuiAutomationClient::Click( switch (type) { case ClickType::kLeft: - request.set_type(yaze::test::ClickRequest::LEFT); + request.set_type( + yaze::test::ClickRequest::CLICK_TYPE_LEFT); break; case ClickType::kRight: - request.set_type(yaze::test::ClickRequest::RIGHT); + request.set_type( + yaze::test::ClickRequest::CLICK_TYPE_RIGHT); break; case ClickType::kMiddle: - request.set_type(yaze::test::ClickRequest::MIDDLE); + request.set_type( + yaze::test::ClickRequest::CLICK_TYPE_MIDDLE); break; case ClickType::kDouble: - request.set_type(yaze::test::ClickRequest::DOUBLE); + request.set_type( + yaze::test::ClickRequest::CLICK_TYPE_DOUBLE); break; } @@ -112,6 +153,7 @@ absl::StatusOr GuiAutomationClient::Click( result.message = response.message(); result.execution_time = std::chrono::milliseconds( response.execution_time_ms()); + result.test_id = response.test_id(); return result; #else return absl::UnimplementedError("gRPC not available"); @@ -145,6 +187,7 @@ absl::StatusOr GuiAutomationClient::Type( result.message = response.message(); result.execution_time = std::chrono::milliseconds( response.execution_time_ms()); + result.test_id = response.test_id(); return result; #else return absl::UnimplementedError("gRPC not available"); @@ -178,6 +221,7 @@ absl::StatusOr GuiAutomationClient::Wait( result.message = response.message(); result.execution_time = std::chrono::milliseconds( response.elapsed_ms()); + result.test_id = response.test_id(); return result; #else return absl::UnimplementedError("gRPC not available"); @@ -210,6 +254,7 @@ absl::StatusOr GuiAutomationClient::Assert( result.actual_value = response.actual_value(); result.expected_value = response.expected_value(); result.execution_time = std::chrono::milliseconds(0); + result.test_id = response.test_id(); return result; #else return absl::UnimplementedError("gRPC not available"); @@ -226,7 +271,8 @@ absl::StatusOr GuiAutomationClient::Screenshot( yaze::test::ScreenshotRequest request; request.set_window_title(""); // Empty = main window request.set_output_path("/tmp/yaze_screenshot.png"); // Default path - request.set_format(yaze::test::ScreenshotRequest::PNG); // Always PNG for now + request.set_format( + yaze::test::ScreenshotRequest::IMAGE_FORMAT_PNG); // Always PNG for now yaze::test::ScreenshotResponse response; grpc::ClientContext context; @@ -242,6 +288,152 @@ absl::StatusOr GuiAutomationClient::Screenshot( result.success = response.success(); result.message = response.message(); result.execution_time = std::chrono::milliseconds(0); + result.test_id.clear(); + return result; +#else + return absl::UnimplementedError("gRPC not available"); +#endif +} + +absl::StatusOr GuiAutomationClient::GetTestStatus( + const std::string& test_id) { +#ifdef YAZE_WITH_GRPC + if (!stub_) { + return absl::FailedPreconditionError("Not connected. Call Connect() first."); + } + + yaze::test::GetTestStatusRequest request; + request.set_test_id(test_id); + + yaze::test::GetTestStatusResponse response; + grpc::ClientContext context; + + grpc::Status status = stub_->GetTestStatus(&context, request, &response); + + if (!status.ok()) { + return absl::InternalError( + absl::StrFormat("GetTestStatus RPC failed: %s", status.error_message())); + } + + TestStatusDetails details; + details.test_id = test_id; + details.status = ConvertStatusProto(response.status()); + details.queued_at = OptionalTimeFromMillis(response.queued_at_ms()); + details.started_at = OptionalTimeFromMillis(response.started_at_ms()); + details.completed_at = OptionalTimeFromMillis(response.completed_at_ms()); + details.execution_time_ms = response.execution_time_ms(); + details.error_message = response.error_message(); + details.assertion_failures.assign(response.assertion_failures().begin(), + response.assertion_failures().end()); + return details; +#else + return absl::UnimplementedError("gRPC not available"); +#endif +} + +absl::StatusOr GuiAutomationClient::ListTests( + const std::string& category_filter, int page_size, + const std::string& page_token) { +#ifdef YAZE_WITH_GRPC + if (!stub_) { + return absl::FailedPreconditionError("Not connected. Call Connect() first."); + } + + yaze::test::ListTestsRequest request; + if (!category_filter.empty()) { + request.set_category_filter(category_filter); + } + if (page_size > 0) { + request.set_page_size(page_size); + } + if (!page_token.empty()) { + request.set_page_token(page_token); + } + + yaze::test::ListTestsResponse response; + grpc::ClientContext context; + + grpc::Status status = stub_->ListTests(&context, request, &response); + + if (!status.ok()) { + return absl::InternalError( + absl::StrFormat("ListTests RPC failed: %s", status.error_message())); + } + + ListTestsResult result; + result.total_count = response.total_count(); + result.next_page_token = response.next_page_token(); + result.tests.reserve(response.tests_size()); + + for (const auto& test_info : response.tests()) { + HarnessTestSummary summary; + summary.test_id = test_info.test_id(); + summary.name = test_info.name(); + summary.category = test_info.category(); + summary.last_run_at = + OptionalTimeFromMillis(test_info.last_run_timestamp_ms()); + summary.total_runs = test_info.total_runs(); + summary.pass_count = test_info.pass_count(); + summary.fail_count = test_info.fail_count(); + summary.average_duration_ms = test_info.average_duration_ms(); + result.tests.push_back(std::move(summary)); + } + + return result; +#else + return absl::UnimplementedError("gRPC not available"); +#endif +} + +absl::StatusOr GuiAutomationClient::GetTestResults( + const std::string& test_id, bool include_logs) { +#ifdef YAZE_WITH_GRPC + if (!stub_) { + return absl::FailedPreconditionError("Not connected. Call Connect() first."); + } + + yaze::test::GetTestResultsRequest request; + request.set_test_id(test_id); + request.set_include_logs(include_logs); + + yaze::test::GetTestResultsResponse response; + grpc::ClientContext context; + + grpc::Status status = stub_->GetTestResults(&context, request, &response); + + if (!status.ok()) { + return absl::InternalError( + absl::StrFormat("GetTestResults RPC failed: %s", + status.error_message())); + } + + TestResultDetails result; + result.test_id = test_id; + result.success = response.success(); + result.test_name = response.test_name(); + result.category = response.category(); + result.executed_at = OptionalTimeFromMillis(response.executed_at_ms()); + result.duration_ms = response.duration_ms(); + + result.assertions.reserve(response.assertions_size()); + for (const auto& assertion : response.assertions()) { + AssertionOutcome outcome; + outcome.description = assertion.description(); + outcome.passed = assertion.passed(); + outcome.expected_value = assertion.expected_value(); + outcome.actual_value = assertion.actual_value(); + outcome.error_message = assertion.error_message(); + result.assertions.push_back(std::move(outcome)); + } + + if (include_logs) { + result.logs.assign(response.logs().begin(), response.logs().end()); + } + + for (const auto& metric : response.metrics()) { + result.metrics.emplace(metric.first, metric.second); + } + return result; #else return absl::UnimplementedError("gRPC not available"); diff --git a/src/cli/service/gui_automation_client.h b/src/cli/service/gui_automation_client.h index 8a3aecb1..a613cd06 100644 --- a/src/cli/service/gui_automation_client.h +++ b/src/cli/service/gui_automation_client.h @@ -6,9 +6,12 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/time/time.h" #include +#include #include +#include #include #include @@ -39,6 +42,82 @@ struct AutomationResult { std::chrono::milliseconds execution_time; std::string actual_value; // For assertions std::string expected_value; // For assertions + std::string test_id; // Test execution identifier (for introspection) +}; + +/** + * @brief Execution status codes returned by the harness + */ +enum class TestRunStatus { + kUnknown, + kQueued, + kRunning, + kPassed, + kFailed, + kTimeout +}; + +/** + * @brief Detailed information about an individual test execution + */ +struct TestStatusDetails { + std::string test_id; + TestRunStatus status = TestRunStatus::kUnknown; + std::optional queued_at; + std::optional started_at; + std::optional completed_at; + int execution_time_ms = 0; + std::string error_message; + std::vector assertion_failures; +}; + +/** + * @brief Aggregated metadata about a harness test + */ +struct HarnessTestSummary { + std::string test_id; + std::string name; + std::string category; + std::optional last_run_at; + int total_runs = 0; + int pass_count = 0; + int fail_count = 0; + int average_duration_ms = 0; +}; + +/** + * @brief Result container for ListTests RPC + */ +struct ListTestsResult { + std::vector tests; + std::string next_page_token; + int total_count = 0; +}; + +/** + * @brief Individual assertion outcome within a harness test + */ +struct AssertionOutcome { + std::string description; + bool passed = false; + std::string expected_value; + std::string actual_value; + std::string error_message; +}; + +/** + * @brief Detailed execution results for a specific harness test + */ +struct TestResultDetails { + std::string test_id; + bool success = false; + std::string test_name; + std::string category; + std::optional executed_at; + int duration_ms = 0; + std::vector assertions; + std::vector logs; + std::map metrics; }; /** @@ -128,6 +207,24 @@ class GuiAutomationClient { absl::StatusOr Screenshot(const std::string& region = "full", const std::string& format = "PNG"); + /** + * @brief Fetch the current execution status for a harness test + */ + absl::StatusOr GetTestStatus(const std::string& test_id); + + /** + * @brief Enumerate harness tests with optional filtering + */ + absl::StatusOr ListTests(const std::string& category_filter = "", + int page_size = 100, + const std::string& page_token = ""); + + /** + * @brief Retrieve detailed results for a harness test execution + */ + absl::StatusOr GetTestResults(const std::string& test_id, + bool include_logs = false); + /** * @brief Check if client is connected */