diff --git a/src/cli/handlers/agent/test_commands.cc b/src/cli/handlers/agent/test_commands.cc index 74c3f4e5..59b49736 100644 --- a/src/cli/handlers/agent/test_commands.cc +++ b/src/cli/handlers/agent/test_commands.cc @@ -1,2351 +1,54 @@ #include "cli/handlers/agent/commands.h" -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include #include #include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/ascii.h" -#include "absl/strings/cord.h" -#include "absl/strings/match.h" -#include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/strings/str_join.h" -#include "absl/strings/str_split.h" -#include "absl/strings/string_view.h" -#include "absl/strings/strip.h" -#include "absl/time/clock.h" -#include "absl/time/time.h" -#include "cli/handlers/agent/common.h" -#include "cli/service/gui_automation_client.h" -#include "cli/service/test_suite.h" -#include "cli/service/test_suite_loader.h" -#include "cli/service/test_suite_reporter.h" -#include "cli/service/test_suite_writer.h" -#include "cli/service/test_workflow_generator.h" -#include "util/macro.h" - -#if defined(_WIN32) -#include -#else -#include -#endif namespace yaze { namespace cli { namespace agent { -namespace { - -constexpr char kExitCodePayloadKey[] = "yaze.cli.exit_code"; - -void AttachExitCode(absl::Status* status, int exit_code) { - if (!status || status->ok()) { - return; - } - status->SetPayload(kExitCodePayloadKey, - absl::Cord(std::to_string(exit_code))); -} - -std::string TrimWhitespace(absl::string_view value) { - return std::string(absl::StripAsciiWhitespace(value)); -} - -bool IsInteractiveInput() { -#if defined(_WIN32) - return _isatty(_fileno(stdin)) != 0; -#else - return isatty(fileno(stdin)) != 0; -#endif -} - -std::string PromptWithDefault(const std::string& prompt, - const std::string& default_value, - bool allow_empty = true) { - while (true) { - std::cout << prompt; - if (!default_value.empty()) { - std::cout << " [" << default_value << "]"; - } - std::cout << ": "; - std::cout.flush(); - - std::string line; - if (!std::getline(std::cin, line)) { - return default_value; - } - std::string trimmed = TrimWhitespace(line); - if (!trimmed.empty()) { - return trimmed; - } - if (!default_value.empty()) { - return default_value; - } - if (allow_empty) { - return std::string(); - } - std::cout << " Value is required." << std::endl; - } -} - -std::string PromptRequired(const std::string& prompt, - const std::string& default_value = std::string()) { - return PromptWithDefault(prompt, default_value, /*allow_empty=*/false); -} - -int PromptInt(const std::string& prompt, int default_value, int min_value) { - while (true) { - std::string default_str = absl::StrCat(default_value); - std::string input = PromptWithDefault(prompt, default_str); - if (input.empty()) { - return default_value; - } - int value = 0; - if (absl::SimpleAtoi(input, &value) && value >= min_value) { - return value; - } - std::cout << " Enter an integer >= " << min_value << "." << std::endl; - } -} - -bool PromptYesNo(const std::string& prompt, bool default_value) { - while (true) { - std::cout << prompt << " [" << (default_value ? "Y/n" : "y/N") - << "]: "; - std::cout.flush(); - std::string line; - if (!std::getline(std::cin, line)) { - return default_value; - } - std::string trimmed = TrimWhitespace(line); - if (trimmed.empty()) { - return default_value; - } - char c = static_cast(std::tolower(static_cast(trimmed[0]))); - if (c == 'y') { - return true; - } - if (c == 'n') { - return false; - } - std::cout << " Please respond with 'y' or 'n'." << std::endl; - } -} - -std::vector ParseCommaSeparated(absl::string_view input) { - std::vector values; - for (absl::string_view token : absl::StrSplit(input, ',')) { - std::string trimmed = TrimWhitespace(token); - if (!trimmed.empty()) { - values.push_back(trimmed); - } - } - return values; -} - -bool ParseKeyValueEntry(const std::string& input, std::string* key, - std::string* value) { - size_t equals = input.find('='); - if (equals == std::string::npos) { - return false; - } - *key = TrimWhitespace(absl::string_view(input.data(), equals)); - *value = TrimWhitespace(absl::string_view(input.data() + equals + 1, - input.size() - equals - 1)); - return !key->empty(); -} - -std::string DeriveTestNameFromPath(const std::string& path) { - if (path.empty()) { - return ""; - } - std::filesystem::path fs_path(path); - std::string stem = fs_path.stem().string(); - if (!stem.empty()) { - return stem; - } - return path; -} - -std::string OutcomeToLabel(TestCaseOutcome outcome) { - switch (outcome) { - case TestCaseOutcome::kPassed: - return "PASS"; - case TestCaseOutcome::kFailed: - return "FAIL"; - case TestCaseOutcome::kError: - return "ERROR"; - case TestCaseOutcome::kSkipped: - return "SKIP"; - } - return "UNKNOWN"; -} - -std::string BuildWidgetCatalogJson(const DiscoverWidgetsResult& catalog) { - std::ostringstream oss; - oss << "{\n"; - oss << " \"generated_at_ms\": "; - if (catalog.generated_at.has_value()) { - oss << absl::ToUnixMillis(catalog.generated_at.value()); - } else { - oss << "null"; - } - oss << ",\n"; - oss << " \"total_widgets\": " << catalog.total_widgets << ",\n"; - oss << " \"windows\": [\n"; - for (size_t w = 0; w < catalog.windows.size(); ++w) { - const auto& window = catalog.windows[w]; - oss << " {\n"; - oss << " \"name\": \"" << JsonEscape(window.name) << "\",\n"; - oss << " \"visible\": " << (window.visible ? "true" : "false") - << ",\n"; - oss << " \"widgets\": [\n"; - for (size_t i = 0; i < window.widgets.size(); ++i) { - const auto& widget = window.widgets[i]; - oss << " {\n"; - oss << " \"path\": \"" << JsonEscape(widget.path) << "\",\n"; - oss << " \"label\": \"" << JsonEscape(widget.label) - << "\",\n"; - oss << " \"type\": \"" << JsonEscape(widget.type) << "\",\n"; - oss << " \"description\": \"" - << JsonEscape(widget.description) << "\",\n"; - oss << " \"suggested_action\": \"" - << JsonEscape(widget.suggested_action) << "\",\n"; - oss << " \"visible\": " - << (widget.visible ? "true" : "false") << ",\n"; - oss << " \"enabled\": " - << (widget.enabled ? "true" : "false") << ",\n"; - oss << " \"widget_id\": " << widget.widget_id << ",\n"; - oss << " \"last_seen_frame\": " << widget.last_seen_frame - << ",\n"; - oss << " \"last_seen_at_ms\": "; - if (widget.last_seen_at.has_value()) { - oss << absl::ToUnixMillis(widget.last_seen_at.value()); - } else { - oss << "null"; - } - oss << ",\n"; - oss << " \"stale\": " - << (widget.stale ? "true" : "false") << ",\n"; - oss << " \"bounds\": "; - if (widget.has_bounds) { - oss << "{\"min\": [" << widget.bounds.min_x << ", " - << widget.bounds.min_y << "], \"max\": [" << widget.bounds.max_x - << ", " << widget.bounds.max_y << "]}"; - } else { - oss << "null"; - } - oss << "\n }"; - if (i + 1 < window.widgets.size()) { - oss << ','; - } - oss << "\n"; - } - oss << " ]\n"; - oss << " }"; - if (w + 1 < catalog.windows.size()) { - oss << ','; - } - oss << "\n"; - } - oss << " ]\n"; - oss << "}\n"; - return oss.str(); -} - -absl::Status WriteWidgetCatalog(const DiscoverWidgetsResult& catalog, - const std::string& output_path) { - std::filesystem::path path(output_path); - if (path.has_parent_path()) { - std::error_code ec; - std::filesystem::create_directories(path.parent_path(), ec); - if (ec) { - return absl::InternalError( - absl::StrCat("Failed to create directories for widget catalog: ", - ec.message())); - } - } - std::ofstream out(path); - if (!out.is_open()) { - return absl::InternalError( - absl::StrCat("Unable to open widget catalog path '", output_path, - "'")); - } - out << BuildWidgetCatalogJson(catalog); - return absl::OkStatus(); -} - -struct ReplayCommandOptions { - std::string script_path; - std::string host = "localhost"; - int port = 50052; - bool ci_mode = false; - std::string output_format = "text"; - std::map parameters; -}; - -absl::StatusOr ParseReplayArgs( - const std::vector& args) { - ReplayCommandOptions options; - - auto parse_int = [](absl::string_view value, - const char* flag) -> absl::StatusOr { - int result = 0; - if (!absl::SimpleAtoi(value, &result)) { - return absl::InvalidArgumentError( - absl::StrCat(flag, " requires an integer value")); - } - if (result <= 0 || result > 65535) { - return absl::InvalidArgumentError( - absl::StrCat(flag, " must be between 1 and 65535")); - } - return result; - }; - - for (size_t i = 0; i < args.size(); ++i) { - const std::string& token = args[i]; - - if (token == "--ci-mode" || token == "--ci") { - options.ci_mode = true; - continue; - } - - if (token == "--host" && i + 1 < args.size()) { - options.host = args[++i]; - continue; - } - if (absl::StartsWith(token, "--host=")) { - options.host = token.substr(7); - continue; - } - - if (token == "--port" && i + 1 < args.size()) { - ASSIGN_OR_RETURN(options.port, parse_int(args[++i], "--port")); - continue; - } - if (absl::StartsWith(token, "--port=")) { - ASSIGN_OR_RETURN(options.port, - parse_int(token.substr(7), "--port")); - continue; - } - - if ((token == "--format" || token == "--output") && - i + 1 < args.size()) { - options.output_format = absl::AsciiStrToLower(args[++i]); - continue; - } - if (absl::StartsWith(token, "--format=") || - absl::StartsWith(token, "--output=")) { - options.output_format = - absl::AsciiStrToLower(token.substr(token.find('=') + 1)); - continue; - } - - if (token == "--param" && i + 1 < args.size()) { - std::string pair = args[++i]; - auto eq = pair.find('='); - if (eq == std::string::npos) { - return absl::InvalidArgumentError( - "--param expects KEY=VALUE format"); - } - options.parameters[pair.substr(0, eq)] = pair.substr(eq + 1); - continue; - } - if (absl::StartsWith(token, "--param=")) { - std::string pair = token.substr(8); - auto eq = pair.find('='); - if (eq == std::string::npos) { - return absl::InvalidArgumentError( - "--param expects KEY=VALUE format"); - } - options.parameters[pair.substr(0, eq)] = pair.substr(eq + 1); - continue; - } - - if (token == "--script" && i + 1 < args.size()) { - options.script_path = args[++i]; - continue; - } - if (absl::StartsWith(token, "--script=")) { - options.script_path = token.substr(9); - continue; - } - - if (absl::StartsWith(token, "--")) { - return absl::InvalidArgumentError( - absl::StrCat("Unknown flag for agent test replay: ", token)); - } - - if (options.script_path.empty()) { - options.script_path = token; - continue; - } - +absl::Status HandleTestCommand(const std::vector& args) { + if (args.empty()) { return absl::InvalidArgumentError( - absl::StrCat("Unexpected argument: ", token)); - } - - if (options.script_path.empty()) { - return absl::InvalidArgumentError( - "Usage: agent test replay [--ci-mode] [--host ] " - "[--port ] [--format text|json] [--param KEY=VALUE]"); - } - - if (options.output_format != "text" && options.output_format != "json") { - return absl::InvalidArgumentError( - "--format must be either 'text' or 'json'"); - } - - return options; -} - -void PrintReplayTextSummary(const ReplayCommandOptions& options, - const ReplayTestResult& result) { - std::cout << "\n=== Replay Test ===\n"; - std::cout << "Script: " << options.script_path << "\n"; - std::cout << "Server: " << HarnessAddress(options.host, options.port) - << "\n"; - if (!options.parameters.empty()) { - std::cout << "Parameters:\n"; - for (const auto& [key, value] : options.parameters) { - std::cout << " • " << key << "=" << value << "\n"; - } - } - std::cout << "Steps Executed: " << result.steps_executed << "\n"; - if (!result.replay_session_id.empty()) { - std::cout << "Replay Session: " << result.replay_session_id << "\n"; - } - if (result.success) { - std::cout << "✅ Replay succeeded\n"; - } else { - std::cout << "❌ Replay failed: " << result.message << "\n"; - } - if (!result.assertions.empty()) { - std::cout << "Assertions (" << result.assertions.size() << "):\n"; - for (const auto& assertion : result.assertions) { - std::cout << " - " << assertion.description << ": " - << (assertion.passed ? "PASS" : "FAIL"); - if (!assertion.error_message.empty()) { - std::cout << " (" << assertion.error_message << ")"; - } - std::cout << "\n"; - } - } - if (!result.logs.empty()) { - std::cout << "Logs:\n"; - for (const auto& log : result.logs) { - std::cout << " • " << log << "\n"; - } - } -} - -void PrintReplayJsonSummary(const ReplayCommandOptions& options, - const ReplayTestResult& result) { - std::cout << "{\n"; - std::cout << " \"script_path\": \"" << JsonEscape(options.script_path) - << "\",\n"; - std::cout << " \"host\": \"" << JsonEscape(options.host) << "\",\n"; - std::cout << " \"port\": " << options.port << ",\n"; - std::cout << " \"ci_mode\": " << (options.ci_mode ? "true" : "false") - << ",\n"; - std::cout << " \"parameters\": {"; - size_t param_index = 0; - for (const auto& [key, value] : options.parameters) { - if (param_index > 0) { - std::cout << ", "; - } - std::cout << "\"" << JsonEscape(key) << "\": \"" - << JsonEscape(value) << "\""; - ++param_index; - } - std::cout << "},\n"; - std::cout << " \"success\": " << (result.success ? "true" : "false") - << ",\n"; - std::cout << " \"message\": \"" << JsonEscape(result.message) - << "\",\n"; - std::cout << " \"steps_executed\": " << result.steps_executed - << ",\n"; - if (result.replay_session_id.empty()) { - std::cout << " \"replay_session_id\": null,\n"; - } else { - std::cout << " \"replay_session_id\": \"" - << JsonEscape(result.replay_session_id) << "\",\n"; - } - std::cout << " \"assertions\": [\n"; - for (size_t i = 0; i < result.assertions.size(); ++i) { - const auto& assertion = result.assertions[i]; - std::cout << " {\"description\": \"" - << JsonEscape(assertion.description) << "\", \"passed\": " - << (assertion.passed ? "true" : "false"); - if (!assertion.error_message.empty()) { - std::cout << ", \"error\": \"" - << JsonEscape(assertion.error_message) << "\""; - } - std::cout << "}"; - if (i + 1 < result.assertions.size()) { - std::cout << ','; - } - std::cout << "\n"; - } - std::cout << " ],\n"; - std::cout << " \"logs\": [\n"; - for (size_t i = 0; i < result.logs.size(); ++i) { - std::cout << " \"" << JsonEscape(result.logs[i]) << "\""; - if (i + 1 < result.logs.size()) { - std::cout << ','; - } - std::cout << "\n"; - } - std::cout << " ]\n"; - std::cout << "}\n"; -} - -absl::Status HandleTestReplayCommand(const std::vector& arg_vec) { - ASSIGN_OR_RETURN(auto options, ParseReplayArgs(arg_vec)); - - bool text_output = options.output_format == "text"; - bool json_output = options.output_format == "json"; - -#ifndef YAZE_WITH_GRPC - std::string error = - "GUI automation requires YAZE_WITH_GRPC=ON at build time.\n" - "Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON"; - ReplayTestResult result; - result.success = false; - result.message = error; - if (json_output) { - PrintReplayJsonSummary(options, result); - } else { - PrintReplayTextSummary(options, result); - } - absl::Status status = absl::UnimplementedError(error); - AttachExitCode(&status, 2); - return status; -#else - GuiAutomationClient client(HarnessAddress(options.host, options.port)); - auto connect_status = client.Connect(); - if (!connect_status.ok()) { - std::string formatted_error = absl::StrFormat( - "Failed to connect to test harness at %s:%d -- %s", options.host, - options.port, connect_status.message()); - ReplayTestResult result; - result.success = false; - result.message = formatted_error; - if (json_output) { - PrintReplayJsonSummary(options, result); - } else { - PrintReplayTextSummary(options, result); - } - absl::Status status = absl::UnavailableError(formatted_error); - AttachExitCode(&status, 2); - return status; - } - - ASSIGN_OR_RETURN(ReplayTestResult result, - client.ReplayTest(options.script_path, options.ci_mode, - options.parameters)); - - if (json_output) { - PrintReplayJsonSummary(options, result); - } else { - PrintReplayTextSummary(options, result); - } - - if (!result.success) { - absl::Status status = absl::InternalError(result.message); - AttachExitCode(&status, options.ci_mode ? 2 : 1); - return status; - } - - return absl::OkStatus(); -#endif -} - -absl::Status HandleTestRunCommand(const std::vector& arg_vec) { - std::string prompt; - std::string host = "localhost"; - int port = 50052; - int timeout_sec = 30; - std::string output_format = "text"; - - for (size_t i = 0; i < arg_vec.size(); ++i) { - const std::string& token = arg_vec[i]; - - if (token == "--prompt" && i + 1 < arg_vec.size()) { - prompt = arg_vec[++i]; - } else if (token == "--host" && i + 1 < arg_vec.size()) { - host = arg_vec[++i]; - } else if (token == "--port" && i + 1 < arg_vec.size()) { - port = std::stoi(arg_vec[++i]); - } else if (token == "--timeout" && i + 1 < arg_vec.size()) { - timeout_sec = std::stoi(arg_vec[++i]); - } else if (token == "--output" && i + 1 < arg_vec.size()) { - output_format = arg_vec[++i]; - } else if (absl::StartsWith(token, "--prompt=")) { - prompt = token.substr(9); - } else if (absl::StartsWith(token, "--host=")) { - host = token.substr(7); - } else if (absl::StartsWith(token, "--port=")) { - port = std::stoi(token.substr(7)); - } else if (absl::StartsWith(token, "--timeout=")) { - timeout_sec = std::stoi(token.substr(10)); - } else if (absl::StartsWith(token, "--output=")) { - output_format = token.substr(9); - } - } - - if (prompt.empty()) { - return absl::InvalidArgumentError( - "Usage: agent test --prompt \"\" [--host ] [--port " - "] [--timeout ] [--output text|json|yaml]\n\n" - "Examples:\n" - " z3ed agent test --prompt \"Open Overworld editor\"\n" - " z3ed agent test --prompt \"Open Dungeon editor and verify it " - "loads\"\n" - " z3ed agent test --prompt \"Click Open ROM button\" --output json"); - } - - output_format = absl::AsciiStrToLower(output_format); - bool text_output = (output_format == "text" || output_format == "human"); - bool json_output = (output_format == "json"); - bool yaml_output = (output_format == "yaml"); - if (!text_output && !json_output && !yaml_output) { - return absl::InvalidArgumentError( - "--output must be one of: text, json, yaml"); - } - bool machine_output = !text_output; - -#ifndef YAZE_WITH_GRPC - std::string error = - "GUI automation requires YAZE_WITH_GRPC=ON at build time.\n" - "Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON"; - if (machine_output) { - if (json_output) { - std::cout << "{\n" - << " \"prompt\": \"" << JsonEscape(prompt) << "\",\n" - << " \"success\": false,\n" - << " \"error\": \"" << JsonEscape(error) << "\"\n" - << "}\n"; - } else { - std::cout << "prompt: " << YamlQuote(prompt) << "\n" - << "success: false\n" - << "error: " << YamlQuote(error) << "\n"; - } - } else { - std::cout << error << std::endl; - } - return absl::UnimplementedError(error); -#else - struct StepSummary { - std::string description; - bool success = false; - int64_t duration_ms = 0; - std::string message; - std::string test_id; - }; - - std::vector step_summaries; - std::vector emitted_test_ids; - std::chrono::steady_clock::time_point start_time; - bool timer_started = false; - - auto EmitMachineSummary = [&](bool success, absl::string_view error_message, - int64_t elapsed_override_ms = -1) { - if (!machine_output) { - return; - } - - int64_t elapsed_ms = elapsed_override_ms; - if (elapsed_ms < 0) { - if (timer_started) { - elapsed_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - start_time) - .count(); - } else { - elapsed_ms = 0; - } - } - - std::string primary_test_id = - emitted_test_ids.empty() ? "" : emitted_test_ids.back(); - - if (json_output) { - std::cout << "{\n"; - std::cout << " \"prompt\": \"" << JsonEscape(prompt) << "\",\n"; - std::cout << " \"host\": \"" << JsonEscape(host) << "\",\n"; - std::cout << " \"port\": " << port << ",\n"; - std::cout << " \"success\": " << (success ? "true" : "false") << ",\n"; - std::cout << " \"timeout_seconds\": " << timeout_sec << ",\n"; - if (!primary_test_id.empty()) { - std::cout << " \"test_id\": \"" << JsonEscape(primary_test_id) - << "\",\n"; - } else { - std::cout << " \"test_id\": null,\n"; - } - std::cout << " \"test_ids\": ["; - for (size_t i = 0; i < emitted_test_ids.size(); ++i) { - if (i > 0) { - std::cout << ", "; - } - std::cout << "\"" << JsonEscape(emitted_test_ids[i]) << "\""; - } - std::cout << "],\n"; - std::cout << " \"elapsed_ms\": " << elapsed_ms << ",\n"; - std::cout << " \"steps\": [\n"; - for (size_t i = 0; i < step_summaries.size(); ++i) { - const auto& step = step_summaries[i]; - std::string message_json = - step.message.empty() - ? "null" - : absl::StrCat("\"", JsonEscape(step.message), "\""); - std::string test_id_json = - step.test_id.empty() - ? "null" - : absl::StrCat("\"", JsonEscape(step.test_id), "\""); - - struct SuiteRunOptions { - std::string suite_path; - std::string host = "localhost"; - int port = 50052; - bool ci_mode = false; - bool stop_on_failure = false; - std::string output_format = "text"; - std::vector group_filters; - std::vector tag_filters; - std::map parameter_overrides; - std::optional retry_override; - std::string junit_output_path; - }; - - void AppendCsvList(absl::string_view csv, - std::vector* output) { - for (absl::string_view part : absl::StrSplit(csv, ',', absl::SkipEmpty())) { - std::string value = std::string(absl::StripAsciiWhitespace(part)); - if (!value.empty()) { - output->push_back(value); - } - } - } - - absl::StatusOr ParseSuiteRunArgs( - const std::vector& args) { - SuiteRunOptions options; - - auto parse_int = [](absl::string_view value, - const char* flag) -> absl::StatusOr { - int result = 0; - if (!absl::SimpleAtoi(value, &result)) { - return absl::InvalidArgumentError( - absl::StrCat(flag, " requires an integer value")); - } - if (result <= 0 || result > 65535) { - return absl::InvalidArgumentError( - absl::StrCat(flag, " must be between 1 and 65535")); - } - return result; - }; - - for (size_t i = 0; i < args.size(); ++i) { - const std::string& token = args[i]; - - if (token == "--ci-mode" || token == "--ci") { - options.ci_mode = true; - options.stop_on_failure = true; - continue; - } - if (token == "--stop-on-failure") { - options.stop_on_failure = true; - continue; - } - - if ((token == "--host" || token == "-H") && i + 1 < args.size()) { - options.host = args[++i]; - continue; - } - if (absl::StartsWith(token, "--host=")) { - options.host = token.substr(7); - continue; - } - - if ((token == "--port" || token == "-p") && i + 1 < args.size()) { - ASSIGN_OR_RETURN(options.port, parse_int(args[++i], "--port")); - continue; - } - if (absl::StartsWith(token, "--port=")) { - ASSIGN_OR_RETURN(options.port, - parse_int(token.substr(7), "--port")); - continue; - } - - if ((token == "--format" || token == "--output") && - i + 1 < args.size()) { - options.output_format = absl::AsciiStrToLower(args[++i]); - continue; - } - if (absl::StartsWith(token, "--format=") || - absl::StartsWith(token, "--output=")) { - options.output_format = absl::AsciiStrToLower( - token.substr(token.find('=') + 1)); - continue; - } - - if ((token == "--group" || token == "-g") && i + 1 < args.size()) { - AppendCsvList(args[++i], &options.group_filters); - continue; - } - if (absl::StartsWith(token, "--group=")) { - AppendCsvList(token.substr(8), &options.group_filters); - continue; - } - - if ((token == "--tag" || token == "-t") && i + 1 < args.size()) { - AppendCsvList(args[++i], &options.tag_filters); - continue; - } - if (absl::StartsWith(token, "--tag=")) { - AppendCsvList(token.substr(6), &options.tag_filters); - continue; - } - - if (token == "--param" && i + 1 < args.size()) { - std::string pair = args[++i]; - auto eq = pair.find('='); - if (eq == std::string::npos) { - return absl::InvalidArgumentError( - "--param expects KEY=VALUE format"); - } - options.parameter_overrides[pair.substr(0, eq)] = pair.substr(eq + 1); - continue; - } - if (absl::StartsWith(token, "--param=")) { - std::string pair = token.substr(8); - auto eq = pair.find('='); - if (eq == std::string::npos) { - return absl::InvalidArgumentError( - "--param expects KEY=VALUE format"); - } - options.parameter_overrides[pair.substr(0, eq)] = pair.substr(eq + 1); - continue; - } - - if ((token == "--retries" || token == "--retry") && - i + 1 < args.size()) { - int value = 0; - if (!absl::SimpleAtoi(args[++i], &value) || value < 0) { - return absl::InvalidArgumentError( - "--retries expects a non-negative integer"); - } - options.retry_override = value; - continue; - } - if (absl::StartsWith(token, "--retries=") || - absl::StartsWith(token, "--retry=")) { - std::string value = token.substr(token.find('=') + 1); - int retries = 0; - if (!absl::SimpleAtoi(value, &retries) || retries < 0) { - return absl::InvalidArgumentError( - "--retries expects a non-negative integer"); - } - options.retry_override = retries; - continue; - } - - if ((token == "--junit" || token == "--junit-output") && - i + 1 < args.size()) { - options.junit_output_path = args[++i]; - continue; - } - if (absl::StartsWith(token, "--junit=") || - absl::StartsWith(token, "--junit-output=")) { - options.junit_output_path = token.substr(token.find('=') + 1); - continue; - } - - if (token == "--suite" && i + 1 < args.size()) { - options.suite_path = args[++i]; - continue; - } - if (absl::StartsWith(token, "--suite=")) { - options.suite_path = token.substr(8); - continue; - } - - if (!absl::StartsWith(token, "--") && options.suite_path.empty()) { - options.suite_path = token; - continue; - } - - if (!absl::StartsWith(token, "--")) { - return absl::InvalidArgumentError( - absl::StrCat("Unexpected argument: ", token)); - } - - return absl::InvalidArgumentError( - absl::StrCat("Unknown flag for agent test suite run: ", token)); - } - - if (options.suite_path.empty()) { - return absl::InvalidArgumentError( - "Usage: agent test suite run [--group ] [--tag " - "] [--ci-mode] [--format text|json] [--junit ]" - " [--param KEY=VALUE] [--retries N]"); - } - - options.output_format = absl::AsciiStrToLower(options.output_format); - if (options.output_format != "text" && options.output_format != "json") { - return absl::InvalidArgumentError( - "--format must be either 'text' or 'json'"); - } - - return options; - } - - bool MatchesFilter(const std::vector& filters, - absl::string_view value) { - if (filters.empty()) { - return true; - } - for (const auto& filter : filters) { - if (absl::EqualsIgnoreCase(filter, value)) { - return true; - } - } - return false; - } - - bool ShouldRunGroup(const TestGroupDefinition& group, - const SuiteRunOptions& options) { - return MatchesFilter(options.group_filters, group.name); - } - - bool ShouldRunTest(const TestCaseDefinition& test, - const SuiteRunOptions& options) { - if (options.tag_filters.empty()) { - return true; - } - for (const auto& tag : test.tags) { - for (const auto& filter : options.tag_filters) { - if (absl::EqualsIgnoreCase(filter, tag)) { - return true; - } - } - } - return false; - } - - std::map MergeParameters( - const TestCaseDefinition& test, const SuiteRunOptions& options) { - std::map merged = test.parameters; - for (const auto& [key, value] : options.parameter_overrides) { - merged[key] = value; - } - return merged; - } - - int DetermineMaxAttempts(const TestSuiteDefinition& suite, - const SuiteRunOptions& options) { - int retries = suite.config.retry_on_failure; - if (options.retry_override.has_value()) { - retries = options.retry_override.value(); - } - if (retries < 0) { - retries = 0; - } - return retries + 1; - } - - void AddResult(TestSuiteRunSummary* summary, TestCaseRunResult result) { - switch (result.outcome) { - case TestCaseOutcome::kPassed: - summary->passed++; - break; - case TestCaseOutcome::kFailed: - summary->failed++; - break; - case TestCaseOutcome::kError: - summary->errors++; - break; - case TestCaseOutcome::kSkipped: - summary->skipped++; - break; - } - summary->results.push_back(std::move(result)); - } - - TestCaseRunResult ExecuteTestCase( - GuiAutomationClient* client, const TestSuiteDefinition& suite, - const TestGroupDefinition& group, const TestCaseDefinition& test, - const SuiteRunOptions& options, int max_attempts) { - TestCaseRunResult result; - result.test = &test; - result.group = &group; - result.outcome = TestCaseOutcome::kError; - result.start_time = absl::Now(); - - std::map parameters = MergeParameters(test, options); - - for (int attempt = 1; attempt <= max_attempts; ++attempt) { - ++result.attempts; - result.retries = attempt - 1; - - absl::StatusOr replay = - client->ReplayTest(test.script_path, options.ci_mode, parameters); - - if (!replay.ok()) { - result.outcome = TestCaseOutcome::kError; - result.message = replay.status().message(); - break; - } - - result.replay_session_id = replay->replay_session_id; - result.assertions = replay->assertions; - result.logs = replay->logs; - result.message = replay->message; - - if (replay->success) { - result.outcome = TestCaseOutcome::kPassed; - break; - } - - result.outcome = TestCaseOutcome::kFailed; - if (attempt < max_attempts) { - continue; - } - break; - } - - result.duration = absl::Now() - result.start_time; - if (result.outcome == TestCaseOutcome::kPassed && result.message.empty()) { - result.message = "Test passed"; - } - return result; - } - - std::string JoinStrings(const std::vector& values, - absl::string_view delimiter) { - if (values.empty()) { - return ""; - } - return absl::StrJoin(values, delimiter); - } - - absl::StatusOr ExecuteTestSuite( - GuiAutomationClient* client, const TestSuiteDefinition& suite, - const SuiteRunOptions& options) { - TestSuiteRunSummary summary; - summary.suite = &suite; - summary.started_at = absl::Now(); - - int max_attempts = DetermineMaxAttempts(suite, options); - std::unordered_map group_success; - bool interrupted = false; - - for (const auto& group : suite.groups) { - bool group_selected = ShouldRunGroup(group, options); - if (!group_selected) { - group_success[group.name] = false; - continue; - } - - bool dependencies_met = true; - std::vector unmet_dependencies; - for (const std::string& dependency : group.depends_on) { - auto it = group_success.find(dependency); - if (it == group_success.end() || !it->second) { - dependencies_met = false; - unmet_dependencies.push_back(dependency); - } - } - - if (!dependencies_met) { - for (const auto& test : group.tests) { - if (!ShouldRunTest(test, options)) { - continue; - } - TestCaseRunResult skipped; - skipped.test = &test; - skipped.group = &group; - skipped.outcome = TestCaseOutcome::kSkipped; - skipped.message = - absl::StrCat("Skipped because dependencies not satisfied: ", - JoinStrings(unmet_dependencies, ", ")); - AddResult(&summary, std::move(skipped)); - } - group_success[group.name] = false; - continue; - } - - bool group_passed = true; - - for (const auto& test : group.tests) { - if (!ShouldRunTest(test, options)) { - TestCaseRunResult skipped; - skipped.test = &test; - skipped.group = &group; - skipped.outcome = TestCaseOutcome::kSkipped; - skipped.message = "Skipped by CLI filter"; - AddResult(&summary, std::move(skipped)); - continue; - } - - if (interrupted) { - TestCaseRunResult skipped; - skipped.test = &test; - skipped.group = &group; - skipped.outcome = TestCaseOutcome::kSkipped; - skipped.message = - "Skipped because stop-on-failure condition was triggered"; - AddResult(&summary, std::move(skipped)); - continue; - } - - TestCaseRunResult result = - ExecuteTestCase(client, suite, group, test, options, max_attempts); - AddResult(&summary, std::move(result)); - - const auto& stored = summary.results.back(); - if (stored.outcome == TestCaseOutcome::kFailed || - stored.outcome == TestCaseOutcome::kError) { - group_passed = false; - if (options.stop_on_failure) { - interrupted = true; - } - } - } - - group_success[group.name] = group_passed; - } - - summary.total_duration = absl::Now() - summary.started_at; - if (summary.results.empty()) { - return absl::InvalidArgumentError( - "No tests were executed. Adjust filters or suite definition."); - } - - return summary; - } - - std::string SanitizeFileComponent(absl::string_view input) { - std::string sanitized; - sanitized.reserve(input.size()); - for (char c : input) { - if (std::isalnum(static_cast(c))) { - sanitized.push_back(c); - } else if (c == '-' || c == '_') { - sanitized.push_back(c); - } else if (c == ' ') { - sanitized.push_back('_'); - } - } - if (sanitized.empty()) { - sanitized = "suite"; - } - return sanitized; - } - - std::string DefaultJUnitOutputPath(const TestSuiteDefinition& suite) { - std::string name = suite.name.empty() ? "suite" : suite.name; - std::string sanitized = SanitizeFileComponent(name); - return absl::StrCat("test-results/junit/", sanitized, ".xml"); - } - - std::string FormatRfc3339(absl::Time time) { - if (time == absl::InfinitePast()) { - return ""; - } - return absl::FormatTime("%Y-%m-%dT%H:%M:%SZ", time, absl::UTCTimeZone()); - } - - std::string BuildSuiteJsonSummary(const TestSuiteRunSummary& summary, - const SuiteRunOptions& options, - absl::string_view junit_path) { - std::ostringstream oss; - std::string suite_name = - summary.suite ? summary.suite->name : "YAZE GUI Test Suite"; - - oss << "{\n"; - oss << " \"suite_name\": \"" << JsonEscape(suite_name) << "\",\n"; - oss << " \"suite_file\": \"" << JsonEscape(options.suite_path) - << "\",\n"; - oss << " \"host\": \"" << JsonEscape(options.host) << "\",\n"; - oss << " \"port\": " << options.port << ",\n"; - oss << " \"ci_mode\": " << (options.ci_mode ? "true" : "false") - << ",\n"; - oss << " \"started_at\": \"" - << JsonEscape(FormatRfc3339(summary.started_at)) << "\",\n"; - oss << " \"duration_seconds\": " - << absl::StrFormat("%.3f", - absl::ToDoubleSeconds(summary.total_duration)) - << ",\n"; - oss << " \"totals\": {\n"; - oss << " \"executed\": " << summary.results.size() << ",\n"; - oss << " \"passed\": " << summary.passed << ",\n"; - oss << " \"failed\": " << summary.failed << ",\n"; - oss << " \"errors\": " << summary.errors << ",\n"; - oss << " \"skipped\": " << summary.skipped << "\n"; - oss << " },\n"; - - oss << " \"parameters\": {\n"; - size_t param_index = 0; - for (const auto& [key, value] : options.parameter_overrides) { - oss << " \"" << JsonEscape(key) << "\": \"" - << JsonEscape(value) << "\""; - if (++param_index < options.parameter_overrides.size()) { - oss << ","; - } - oss << "\n"; - } - oss << " },\n"; - - oss << " \"groups\": [\n"; - for (size_t i = 0; i < summary.results.size(); ++i) { - const auto& result = summary.results[i]; - const std::string group_name = - result.group ? result.group->name : (result.test ? result.test->group_name : ""); - const std::string test_name = - result.test ? result.test->name : "Test"; - oss << " {\n"; - oss << " \"group\": \"" << JsonEscape(group_name) << "\",\n"; - oss << " \"test\": \"" << JsonEscape(test_name) << "\",\n"; - oss << " \"outcome\": \"" - << JsonEscape(OutcomeToLabel(result.outcome)) << "\",\n"; - oss << " \"duration_seconds\": " - << absl::StrFormat("%.3f", absl::ToDoubleSeconds(result.duration)) - << ",\n"; - oss << " \"attempts\": " << result.attempts << ",\n"; - oss << " \"message\": \"" << JsonEscape(result.message) - << "\",\n"; - if (result.replay_session_id.empty()) { - oss << " \"replay_session_id\": null,\n"; - } else { - oss << " \"replay_session_id\": \"" - << JsonEscape(result.replay_session_id) << "\",\n"; - } - oss << " \"assertions\": [\n"; - for (size_t j = 0; j < result.assertions.size(); ++j) { - const auto& assertion = result.assertions[j]; - oss << " {\"description\": \"" - << JsonEscape(assertion.description) << "\", \"passed\": " - << (assertion.passed ? "true" : "false"); - if (!assertion.error_message.empty()) { - oss << ", \"error\": \"" - << JsonEscape(assertion.error_message) << "\""; - } - oss << "}"; - if (j + 1 < result.assertions.size()) { - oss << ","; - } - oss << "\n"; - } - oss << " ],\n"; - oss << " \"logs\": [\n"; - for (size_t j = 0; j < result.logs.size(); ++j) { - oss << " \"" << JsonEscape(result.logs[j]) << "\""; - if (j + 1 < result.logs.size()) { - oss << ","; - } - oss << "\n"; - } - oss << " ]\n"; - oss << " }"; - if (i + 1 < summary.results.size()) { - oss << ","; - } - oss << "\n"; - } - oss << " ],\n"; - - if (junit_path.empty()) { - oss << " \"junit_report\": null\n"; - } else { - oss << " \"junit_report\": \"" << JsonEscape(junit_path) - << "\"\n"; - } - oss << "}\n"; - return oss.str(); - } - std::cout << " {\n"; - std::cout << " \"index\": " << (i + 1) << ",\n"; - std::cout << " \"description\": \"" << JsonEscape(step.description) - << "\",\n"; - std::cout << " \"success\": " << (step.success ? "true" : "false") - << ",\n"; - std::cout << " \"duration_ms\": " << step.duration_ms << ",\n"; - std::cout << " \"message\": " << message_json << ",\n"; - std::cout << " \"test_id\": " << test_id_json << "\n"; - std::cout << " }"; - if (i + 1 < step_summaries.size()) { - std::cout << ","; - } - std::cout << "\n"; - } - std::cout << " ],\n"; - if (!error_message.empty()) { - std::cout << " \"error\": \"" << JsonEscape(std::string(error_message)) - << "\"\n"; - } else { - std::cout << " \"error\": null\n"; - } - std::cout << "}\n"; - } else if (yaml_output) { - std::cout << "prompt: " << YamlQuote(prompt) << "\n"; - std::cout << "host: " << YamlQuote(host) << "\n"; - std::cout << "port: " << port << "\n"; - std::cout << "success: " << (success ? "true" : "false") << "\n"; - std::cout << "timeout_seconds: " << timeout_sec << "\n"; - if (primary_test_id.empty()) { - std::cout << "test_id: null\n"; - } else { - std::cout << "test_id: " << YamlQuote(primary_test_id) << "\n"; - } - if (emitted_test_ids.empty()) { - std::cout << "test_ids: []\n"; - } else { - std::cout << "test_ids:\n"; - for (const auto& id : emitted_test_ids) { - std::cout << " - " << YamlQuote(id) << "\n"; - } - } - std::cout << "elapsed_ms: " << elapsed_ms << "\n"; - if (step_summaries.empty()) { - std::cout << "steps: []\n"; - } else { - std::cout << "steps:\n"; - for (size_t i = 0; i < step_summaries.size(); ++i) { - const auto& step = step_summaries[i]; - std::cout << " - index: " << (i + 1) << "\n"; - std::cout << " description: " << YamlQuote(step.description) - << "\n"; - std::cout << " success: " << (step.success ? "true" : "false") - << "\n"; - std::cout << " duration_ms: " << step.duration_ms << "\n"; - if (step.message.empty()) { - std::cout << " message: null\n"; - } else { - std::cout << " message: " << YamlQuote(step.message) << "\n"; - } - if (step.test_id.empty()) { - std::cout << " test_id: null\n"; - } else { - std::cout << " test_id: " << YamlQuote(step.test_id) << "\n"; - } - } - } - if (!error_message.empty()) { - std::cout << "error: " << YamlQuote(std::string(error_message)) << "\n"; - } else { - std::cout << "error: null\n"; - } - } - }; - - if (text_output) { - std::cout << "\n=== GUI Automation Test ===\n"; - std::cout << "Prompt: " << prompt << "\n"; - std::cout << "Server: " << host << ":" << port << "\n\n"; - } - - TestWorkflowGenerator generator; - auto workflow_or = generator.GenerateWorkflow(prompt); - if (!workflow_or.ok()) { - EmitMachineSummary(false, workflow_or.status().message()); - return workflow_or.status(); - } - auto workflow = workflow_or.value(); - - if (text_output) { - std::cout << "Generated workflow:\n" << workflow.ToString() << "\n"; - } - - GuiAutomationClient client(HarnessAddress(host, port)); - auto connect_status = client.Connect(); - if (!connect_status.ok()) { - std::string formatted_error = absl::StrFormat( - "Failed to connect to test harness at %s:%d\n" - "Make sure YAZE is running with:\n" - " ./yaze --enable_test_harness --test_harness_port=%d " - "--rom_file=\n\n" - "Error: %s", - host, port, port, connect_status.message()); - EmitMachineSummary(false, formatted_error); - return absl::UnavailableError(formatted_error); - } - - if (text_output) { - std::cout << "✓ Connected to test harness\n\n"; - } - - start_time = std::chrono::steady_clock::now(); - timer_started = true; - int step_num = 0; - - for (const auto& step : workflow.steps) { - step_num++; - StepSummary summary; - summary.description = step.ToString(); - - if (text_output) { - std::cout << absl::StrFormat("[%d/%d] %s ... ", step_num, - workflow.steps.size(), summary.description); - std::cout.flush(); - } - - absl::StatusOr result; - - switch (step.type) { - case TestStepType::kClick: - result = client.Click(step.target); - break; - case TestStepType::kType: - result = client.Type(step.target, step.text, step.clear_first); - break; - case TestStepType::kWait: - result = client.Wait(step.condition, step.timeout_ms); - break; - case TestStepType::kAssert: - result = client.Assert(step.condition); - break; - case TestStepType::kScreenshot: - result = client.Screenshot(); - break; - } - - if (!result.ok()) { - summary.success = false; - summary.message = result.status().message(); - step_summaries.push_back(std::move(summary)); - if (text_output) { - std::cout << "✗ FAILED\n"; - } - EmitMachineSummary(false, result.status().message()); - return absl::InternalError(absl::StrFormat("Step %d failed: %s", step_num, - result.status().message())); - } - - summary.duration_ms = result->execution_time.count(); - summary.message = result->message; - - if (!result->success) { - summary.success = false; - if (!result->test_id.empty()) { - summary.test_id = result->test_id; - emitted_test_ids.push_back(result->test_id); - } - step_summaries.push_back(std::move(summary)); - if (text_output) { - std::cout << "✗ FAILED\n"; - std::cout << " Error: " << result->message << "\n"; - } - EmitMachineSummary(false, result->message); - return absl::InternalError( - absl::StrFormat("Step %d failed: %s", step_num, result->message)); - } - - summary.success = true; - if (!result->test_id.empty()) { - summary.test_id = result->test_id; - emitted_test_ids.push_back(result->test_id); - } - step_summaries.push_back(summary); - - if (text_output) { - std::cout << absl::StrFormat("✓ (%lldms)", - result->execution_time.count()); - if (!result->test_id.empty()) { - std::cout << " [Test ID: " << result->test_id << "]"; - } - std::cout << "\n"; - } - } - - auto end_time = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast( - end_time - start_time); - - if (text_output) { - std::cout << "\n✅ Test passed in " << elapsed.count() << "ms\n"; - - if (!emitted_test_ids.empty()) { - std::cout << "Latest Test ID: " << emitted_test_ids.back() << "\n"; - if (emitted_test_ids.size() > 1) { - std::cout << "Captured Test IDs:\n"; - for (const auto& id : emitted_test_ids) { - std::cout << " - " << id << "\n"; - } - } - std::cout << "Use 'z3ed agent test status --test-id " - << emitted_test_ids.back() << "' for live status updates." - << std::endl; - } - } - - EmitMachineSummary(true, /*error_message=*/"", elapsed.count()); - return absl::OkStatus(); -#endif -} - -absl::Status HandleTestStatusCommand(const std::vector& arg_vec) { - std::string host = "localhost"; - int port = 50052; - std::string test_id; - bool follow = false; - int interval_ms = 1000; - - for (size_t i = 0; i < arg_vec.size(); ++i) { - const std::string& token = arg_vec[i]; - - if (token == "--test-id" && i + 1 < arg_vec.size()) { - test_id = arg_vec[++i]; - } else if (absl::StartsWith(token, "--test-id=")) { - test_id = token.substr(10); - } else if (token == "--host" && i + 1 < arg_vec.size()) { - host = arg_vec[++i]; - } else if (absl::StartsWith(token, "--host=")) { - host = token.substr(7); - } else if (token == "--port" && i + 1 < arg_vec.size()) { - port = std::stoi(arg_vec[++i]); - } else if (absl::StartsWith(token, "--port=")) { - port = std::stoi(token.substr(7)); - } else if (token == "--follow") { - follow = true; - } else if ((token == "--interval" || token == "--interval-ms") && - i + 1 < arg_vec.size()) { - interval_ms = std::max(100, std::stoi(arg_vec[++i])); - } else if (absl::StartsWith(token, "--interval=") || - absl::StartsWith(token, "--interval-ms=")) { - size_t prefix = token.find('='); - interval_ms = std::max(100, std::stoi(token.substr(prefix + 1))); - } - } - - if (test_id.empty()) { - return absl::InvalidArgumentError( - "Usage: agent test status --test-id [--follow] [--host ] " - "[--port ] [--interval-ms ]"); + "Usage: agent test \n" + "Subcommands:\n" + " run - Generate and run a GUI automation test\n" + " replay