diff --git a/src/cli/handlers/agent/test_commands.cc b/src/cli/handlers/agent/test_commands.cc index 1adec53a..7168fc9c 100644 --- a/src/cli/handlers/agent/test_commands.cc +++ b/src/cli/handlers/agent/test_commands.cc @@ -1,7 +1,12 @@ #include "cli/handlers/agent/commands.h" #include +#include +#include #include +#include +#include +#include #include #include #include @@ -9,11 +14,18 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" +#include "absl/strings/cord.h" #include "absl/strings/match.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/strip.h" #include "absl/time/time.h" #include "cli/handlers/agent/common.h" #include "cli/service/gui_automation_client.h" +#include "cli/service/test_suite.h" +#include "cli/service/test_suite_loader.h" +#include "cli/service/test_suite_reporter.h" #include "cli/service/test_workflow_generator.h" #include "util/macro.h" @@ -23,6 +35,124 @@ namespace agent { namespace { +constexpr char kExitCodePayloadKey[] = "yaze.cli.exit_code"; + +void AttachExitCode(absl::Status* status, int exit_code) { + if (!status || status->ok()) { + return; + } + status->SetPayload(kExitCodePayloadKey, + absl::Cord(std::to_string(exit_code))); +} + +std::string OutcomeToLabel(TestCaseOutcome outcome) { + switch (outcome) { + case TestCaseOutcome::kPassed: + return "PASS"; + case TestCaseOutcome::kFailed: + return "FAIL"; + case TestCaseOutcome::kError: + return "ERROR"; + case TestCaseOutcome::kSkipped: + return "SKIP"; + } + return "UNKNOWN"; +} + +std::string BuildWidgetCatalogJson(const DiscoverWidgetsResult& catalog) { + std::ostringstream oss; + oss << "{\n"; + oss << " \"generated_at_ms\": "; + if (catalog.generated_at.has_value()) { + oss << absl::ToUnixMillis(catalog.generated_at.value()); + } else { + oss << "null"; + } + oss << ",\n"; + oss << " \"total_widgets\": " << catalog.total_widgets << ",\n"; + oss << " \"windows\": [\n"; + for (size_t w = 0; w < catalog.windows.size(); ++w) { + const auto& window = catalog.windows[w]; + oss << " {\n"; + oss << " \"name\": \"" << JsonEscape(window.name) << "\",\n"; + oss << " \"visible\": " << (window.visible ? "true" : "false") + << ",\n"; + oss << " \"widgets\": [\n"; + for (size_t i = 0; i < window.widgets.size(); ++i) { + const auto& widget = window.widgets[i]; + oss << " {\n"; + oss << " \"path\": \"" << JsonEscape(widget.path) << "\",\n"; + oss << " \"label\": \"" << JsonEscape(widget.label) + << "\",\n"; + oss << " \"type\": \"" << JsonEscape(widget.type) << "\",\n"; + oss << " \"description\": \"" + << JsonEscape(widget.description) << "\",\n"; + oss << " \"suggested_action\": \"" + << JsonEscape(widget.suggested_action) << "\",\n"; + oss << " \"visible\": " + << (widget.visible ? "true" : "false") << ",\n"; + oss << " \"enabled\": " + << (widget.enabled ? "true" : "false") << ",\n"; + oss << " \"widget_id\": " << widget.widget_id << ",\n"; + oss << " \"last_seen_frame\": " << widget.last_seen_frame + << ",\n"; + oss << " \"last_seen_at_ms\": "; + if (widget.last_seen_at.has_value()) { + oss << absl::ToUnixMillis(widget.last_seen_at.value()); + } else { + oss << "null"; + } + oss << ",\n"; + oss << " \"stale\": " + << (widget.stale ? "true" : "false") << ",\n"; + oss << " \"bounds\": "; + if (widget.has_bounds) { + oss << "{\"min\": [" << widget.bounds.min_x << ", " + << widget.bounds.min_y << "], \"max\": [" << widget.bounds.max_x + << ", " << widget.bounds.max_y << "]}"; + } else { + oss << "null"; + } + oss << "\n }"; + if (i + 1 < window.widgets.size()) { + oss << ','; + } + oss << "\n"; + } + oss << " ]\n"; + oss << " }"; + if (w + 1 < catalog.windows.size()) { + oss << ','; + } + oss << "\n"; + } + oss << " ]\n"; + oss << "}\n"; + return oss.str(); +} + +absl::Status WriteWidgetCatalog(const DiscoverWidgetsResult& catalog, + const std::string& output_path) { + std::filesystem::path path(output_path); + if (path.has_parent_path()) { + std::error_code ec; + std::filesystem::create_directories(path.parent_path(), ec); + if (ec) { + return absl::InternalError( + absl::StrCat("Failed to create directories for widget catalog: ", + ec.message())); + } + } + std::ofstream out(path); + if (!out.is_open()) { + return absl::InternalError( + absl::StrCat("Unable to open widget catalog path '", output_path, + "'")); + } + out << BuildWidgetCatalogJson(catalog); + return absl::OkStatus(); +} + absl::Status HandleTestRunCommand(const std::vector& arg_vec) { std::string prompt; std::string host = "localhost"; diff --git a/src/cli/service/gui_automation_client.cc b/src/cli/service/gui_automation_client.cc index a2d41ce7..5a7c3226 100644 --- a/src/cli/service/gui_automation_client.cc +++ b/src/cli/service/gui_automation_client.cc @@ -3,6 +3,7 @@ #include "cli/service/gui_automation_client.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/time/time.h" @@ -138,6 +139,53 @@ absl::StatusOr GuiAutomationClient::Ping( #endif } +absl::StatusOr GuiAutomationClient::ReplayTest( + const std::string& script_path, bool ci_mode, + const std::map& parameter_overrides) { +#ifdef YAZE_WITH_GRPC + if (!stub_) { + return absl::FailedPreconditionError("Not connected. Call Connect() first."); + } + + yaze::test::ReplayTestRequest request; + request.set_script_path(script_path); + request.set_ci_mode(ci_mode); + for (const auto& [key, value] : parameter_overrides) { + (*request.mutable_parameter_overrides())[key] = value; + } + + yaze::test::ReplayTestResponse response; + grpc::ClientContext context; + + grpc::Status status = stub_->ReplayTest(&context, request, &response); + if (!status.ok()) { + return absl::InternalError( + absl::StrCat("ReplayTest RPC failed: ", status.error_message())); + } + + ReplayTestResult result; + result.success = response.success(); + result.message = response.message(); + result.replay_session_id = response.replay_session_id(); + result.steps_executed = response.steps_executed(); + result.logs.assign(response.logs().begin(), response.logs().end()); + result.assertions.reserve(response.assertions_size()); + for (const auto& assertion_proto : response.assertions()) { + AssertionOutcome assertion; + assertion.description = assertion_proto.description(); + assertion.passed = assertion_proto.passed(); + assertion.expected_value = assertion_proto.expected_value(); + assertion.actual_value = assertion_proto.actual_value(); + assertion.error_message = assertion_proto.error_message(); + result.assertions.push_back(std::move(assertion)); + } + + return result; +#else + return absl::UnimplementedError("gRPC not available"); +#endif +} + absl::StatusOr GuiAutomationClient::Click( const std::string& target, ClickType type) { #ifdef YAZE_WITH_GRPC diff --git a/src/cli/service/gui_automation_client.h b/src/cli/service/gui_automation_client.h index 5f8c9c1a..49d032de 100644 --- a/src/cli/service/gui_automation_client.h +++ b/src/cli/service/gui_automation_client.h @@ -125,6 +125,15 @@ struct TestResultDetails { std::string widget_state; }; +struct ReplayTestResult { + bool success = false; + std::string message; + std::string replay_session_id; + int steps_executed = 0; + std::vector assertions; + std::vector logs; +}; + enum class WidgetTypeFilter { kUnspecified, kAll, @@ -290,6 +299,10 @@ class GuiAutomationClient { absl::StatusOr DiscoverWidgets( const DiscoverWidgetsQuery& query); + absl::StatusOr ReplayTest( + const std::string& script_path, bool ci_mode, + const std::map& parameter_overrides = {}); + /** * @brief Check if client is connected */ diff --git a/src/cli/service/test_suite.h b/src/cli/service/test_suite.h new file mode 100644 index 00000000..6d71efdb --- /dev/null +++ b/src/cli/service/test_suite.h @@ -0,0 +1,93 @@ +#ifndef YAZE_CLI_SERVICE_TEST_SUITE_H_ +#define YAZE_CLI_SERVICE_TEST_SUITE_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/time/time.h" +#include "cli/service/gui_automation_client.h" + +namespace yaze { +namespace cli { + +struct TestSuiteConfig { + int timeout_seconds = 0; + int retry_on_failure = 0; + bool parallel_execution = false; +}; + +struct TestCaseDefinition { + std::string id; + std::string name; + std::string script_path; + std::string description; + std::string group_name; + std::vector tags; + std::map parameters; +}; + +struct TestGroupDefinition { + std::string name; + std::string description; + std::vector depends_on; + std::vector tests; +}; + +struct TestSuiteDefinition { + std::string name; + std::string description; + std::string version; + TestSuiteConfig config; + std::vector groups; + + const TestGroupDefinition* FindGroup(absl::string_view group_name) const; +}; + +inline const TestGroupDefinition* TestSuiteDefinition::FindGroup( + absl::string_view group_name) const { + for (const auto& group : groups) { + if (group.name == group_name) { + return &group; + } + } + return nullptr; +} + +enum class TestCaseOutcome { + kPassed, + kFailed, + kError, + kSkipped +}; + +struct TestCaseRunResult { + const TestCaseDefinition* test = nullptr; + const TestGroupDefinition* group = nullptr; + TestCaseOutcome outcome = TestCaseOutcome::kError; + absl::Time start_time = absl::InfinitePast(); + absl::Duration duration = absl::ZeroDuration(); + int attempts = 0; + int retries = 0; + std::string message; + std::string replay_session_id; + std::vector assertions; + std::vector logs; +}; + +struct TestSuiteRunSummary { + const TestSuiteDefinition* suite = nullptr; + std::vector results; + absl::Time started_at = absl::Now(); + absl::Duration total_duration = absl::ZeroDuration(); + int passed = 0; + int failed = 0; + int errors = 0; + int skipped = 0; +}; + +} // namespace cli +} // namespace yaze + +#endif // YAZE_CLI_SERVICE_TEST_SUITE_H_ diff --git a/src/cli/service/test_suite_loader.cc b/src/cli/service/test_suite_loader.cc new file mode 100644 index 00000000..92e7f01d --- /dev/null +++ b/src/cli/service/test_suite_loader.cc @@ -0,0 +1,615 @@ +#include "cli/service/test_suite_loader.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/strip.h" +#include "absl/strings/string_view.h" +#include "util/macro.h" + +namespace yaze { +namespace cli { +namespace { + +using ::absl::string_view; + +std::string Trim(string_view value) { + return std::string(absl::StripAsciiWhitespace(value)); +} + +std::string StripComment(string_view line) { + bool in_single_quote = false; + bool in_double_quote = false; + for (size_t i = 0; i < line.size(); ++i) { + char c = line[i]; + if (c == '\'') { + if (!in_double_quote) { + in_single_quote = !in_single_quote; + } + } else if (c == '"') { + if (!in_single_quote) { + in_double_quote = !in_double_quote; + } + } else if (c == '#' && !in_single_quote && !in_double_quote) { + return std::string(line.substr(0, i)); + } + } + return std::string(line); +} + +int CountIndent(string_view line) { + int count = 0; + for (char c : line) { + if (c == ' ') { + ++count; + } else if (c == '\t') { + count += 2; // Treat tab as two spaces for simplicity + } else { + break; + } + } + return count; +} + +bool ParseKeyValue(string_view input, std::string* key, std::string* value) { + size_t colon_pos = input.find(':'); + if (colon_pos == string_view::npos) { + return false; + } + *key = Trim(input.substr(0, colon_pos)); + *value = Trim(input.substr(colon_pos + 1)); + return true; +} + +std::string Unquote(string_view value) { + std::string trimmed = Trim(value); + if (trimmed.size() >= 2) { + if ((trimmed.front() == '"' && trimmed.back() == '"') || + (trimmed.front() == '\'' && trimmed.back() == '\'')) { + return std::string(trimmed.substr(1, trimmed.size() - 2)); + } + } + return trimmed; +} + +std::vector ParseInlineList(string_view value) { + std::vector items; + std::string trimmed = Trim(value); + if (trimmed.empty()) { + return items; + } + if (trimmed.front() != '[' || trimmed.back() != ']') { + items.push_back(Unquote(trimmed)); + return items; + } + string_view inner(trimmed.data() + 1, trimmed.size() - 2); + if (inner.empty()) { + return items; + } + for (string_view piece : absl::StrSplit(inner, ',', absl::SkipWhitespace())) { + items.push_back(Unquote(piece)); + } + return items; +} + +absl::StatusOr ParseInt(string_view value) { + int result = 0; + if (!absl::SimpleAtoi(value, &result)) { + return absl::InvalidArgumentError( + absl::StrCat("Expected integer value, got '", value, "'")); + } + return result; +} + +absl::StatusOr ParseDurationSeconds(string_view value) { + std::string lower = absl::AsciiStrToLower(std::string(value)); + if (lower.empty()) { + return 0; + } + int multiplier = 1; + if (absl::EndsWith(lower, "ms")) { + lower = lower.substr(0, lower.size() - 2); + multiplier = 0; // Use 0 to indicate sub-second resolution + } else if (absl::EndsWith(lower, "s")) { + lower = lower.substr(0, lower.size() - 1); + } else if (absl::EndsWith(lower, "m")) { + lower = lower.substr(0, lower.size() - 1); + multiplier = 60; + } + + int numeric = 0; + if (!absl::SimpleAtoi(lower, &numeric)) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid duration value '", value, "'")); + } + + if (multiplier == 0) { + // Round milliseconds up to nearest whole second + return (numeric + 999) / 1000; + } + return numeric * multiplier; +} + +bool ParseBoolean(string_view value, bool* output) { + std::string lower = absl::AsciiStrToLower(std::string(value)); + if (lower == "true" || lower == "yes" || lower == "on" || lower == "1") { + *output = true; + return true; + } + if (lower == "false" || lower == "no" || lower == "off" || lower == "0") { + *output = false; + return true; + } + return false; +} + +std::string DeriveTestName(const std::string& path) { + std::filesystem::path fs_path(path); + std::string stem = fs_path.stem().string(); + if (stem.empty()) { + return path; + } + return stem; +} + +absl::Status ParseScalarConfig(const std::string& key, const std::string& value, + TestSuiteConfig* config) { + if (key == "timeout_per_test") { + ASSIGN_OR_RETURN(int seconds, ParseDurationSeconds(value)); + config->timeout_seconds = seconds; + return absl::OkStatus(); + } + if (key == "retry_on_failure") { + ASSIGN_OR_RETURN(int retries, ParseInt(value)); + config->retry_on_failure = retries; + return absl::OkStatus(); + } + if (key == "parallel_execution") { + bool enabled = false; + if (!ParseBoolean(value, &enabled)) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid boolean for parallel_execution: '", value, + "'")); + } + config->parallel_execution = enabled; + return absl::OkStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("Unknown config key: '", key, "'")); +} + +absl::Status ParseStringListBlock(const std::vector& lines, + size_t* index, int base_indent, + std::vector* output) { + while (*index < lines.size()) { + std::string raw = StripComment(lines[*index]); + std::string trimmed = Trim(raw); + int indent = CountIndent(raw); + if (trimmed.empty()) { + ++(*index); + continue; + } + if (indent < base_indent) { + break; + } + if (indent != base_indent) { + return absl::InvalidArgumentError( + "Invalid indentation in list block"); + } + if (trimmed.empty() || trimmed.front() != '-') { + return absl::InvalidArgumentError("Expected list entry starting with '-'"); + } + std::string value = Trim(trimmed.substr(1)); + output->push_back(Unquote(value)); + ++(*index); + } + return absl::OkStatus(); +} + +absl::Status ParseParametersBlock(const std::vector& lines, + size_t* index, int base_indent, + std::map* params) { + while (*index < lines.size()) { + std::string raw = StripComment(lines[*index]); + std::string trimmed = Trim(raw); + int indent = CountIndent(raw); + if (trimmed.empty()) { + ++(*index); + continue; + } + if (indent < base_indent) { + break; + } + if (indent != base_indent) { + return absl::InvalidArgumentError("Invalid indentation in parameters block"); + } + std::string key; + std::string value; + if (!ParseKeyValue(trimmed, &key, &value)) { + return absl::InvalidArgumentError( + "Expected key/value pair inside parameters block"); + } + (*params)[key] = Unquote(value); + ++(*index); + } + return absl::OkStatus(); +} + +absl::Status ParseTestCaseEntry(const std::vector& lines, + size_t* index, int base_indent, + TestGroupDefinition* group) { + const std::string& raw_line = lines[*index]; + std::string stripped = StripComment(raw_line); + int indent = CountIndent(stripped); + if (indent != base_indent) { + return absl::InvalidArgumentError("Invalid indentation for test case entry"); + } + + size_t dash_pos = stripped.find('-'); + if (dash_pos == std::string::npos) { + return absl::InvalidArgumentError("Malformed list entry in tests block"); + } + + std::string content = Trim(stripped.substr(dash_pos + 1)); + TestCaseDefinition test; + test.group_name = group->name; + + auto commit_test = [&]() { + if (test.script_path.empty()) { + return absl::InvalidArgumentError("Test case missing script_path"); + } + if (test.name.empty()) { + test.name = DeriveTestName(test.script_path); + } + if (test.id.empty()) { + test.id = absl::StrCat(test.group_name, ":", test.name); + } + group->tests.push_back(std::move(test)); + return absl::OkStatus(); + }; + + if (content.empty()) { + ++(*index); + } else if (content.find(':') == std::string::npos) { + test.script_path = Unquote(content); + ++(*index); + return commit_test(); + } else { + std::string key; + std::string value; + if (!ParseKeyValue(content, &key, &value)) { + return absl::InvalidArgumentError("Malformed key/value in test entry"); + } + if (key == "path" || key == "script" || key == "script_path") { + test.script_path = Unquote(value); + } else if (key == "name") { + test.name = Unquote(value); + } else if (key == "description") { + test.description = Unquote(value); + } else if (key == "id") { + test.id = Unquote(value); + } else if (key == "tags") { + auto tags = ParseInlineList(value); + test.tags.insert(test.tags.end(), tags.begin(), tags.end()); + } else { + test.parameters[key] = Unquote(value); + } + ++(*index); + } + + while (*index < lines.size()) { + std::string raw = StripComment(lines[*index]); + std::string trimmed = Trim(raw); + int indent_next = CountIndent(raw); + if (trimmed.empty()) { + ++(*index); + continue; + } + if (indent_next <= base_indent) { + break; + } + if (indent_next == base_indent + 2) { + std::string key; + std::string value; + if (!ParseKeyValue(trimmed, &key, &value)) { + return absl::InvalidArgumentError( + "Expected key/value pair in test definition"); + } + if (key == "path" || key == "script" || key == "script_path") { + test.script_path = Unquote(value); + } else if (key == "name") { + test.name = Unquote(value); + } else if (key == "description") { + test.description = Unquote(value); + } else if (key == "id") { + test.id = Unquote(value); + } else if (key == "tags") { + auto tags = ParseInlineList(value); + if (tags.empty() && value.empty()) { + ++(*index); + RETURN_IF_ERROR(ParseStringListBlock(lines, index, indent_next + 2, + &test.tags)); + continue; + } + test.tags.insert(test.tags.end(), tags.begin(), tags.end()); + } else if (key == "parameters") { + if (!value.empty()) { + return absl::InvalidArgumentError( + "parameters block must be indented on following lines"); + } + ++(*index); + RETURN_IF_ERROR(ParseParametersBlock(lines, index, indent_next + 2, + &test.parameters)); + continue; + } else { + test.parameters[key] = Unquote(value); + } + ++(*index); + } else { + return absl::InvalidArgumentError( + "Unexpected indentation inside test entry"); + } + } + + return commit_test(); +} + +absl::Status ParseTestsBlock(const std::vector& lines, + size_t* index, int base_indent, + TestGroupDefinition* group) { + while (*index < lines.size()) { + std::string raw = StripComment(lines[*index]); + std::string trimmed = Trim(raw); + int indent = CountIndent(raw); + if (trimmed.empty()) { + ++(*index); + continue; + } + if (indent < base_indent) { + break; + } + if (indent != base_indent) { + return absl::InvalidArgumentError( + "Invalid indentation inside tests block"); + } + RETURN_IF_ERROR(ParseTestCaseEntry(lines, index, base_indent, group)); + } + return absl::OkStatus(); +} + +absl::Status ParseGroupEntry(const std::vector& lines, + size_t* index, TestSuiteDefinition* suite) { + const std::string& raw_line = lines[*index]; + std::string stripped = StripComment(raw_line); + int base_indent = CountIndent(stripped); + + size_t dash_pos = stripped.find('-'); + if (dash_pos == std::string::npos || dash_pos < base_indent) { + return absl::InvalidArgumentError("Expected '-' to start group entry"); + } + + std::string content = Trim(stripped.substr(dash_pos + 1)); + TestGroupDefinition group; + + if (!content.empty()) { + std::string key; + std::string value; + if (!ParseKeyValue(content, &key, &value)) { + return absl::InvalidArgumentError("Malformed group entry"); + } + if (key == "name") { + group.name = Unquote(value); + } else if (key == "description") { + group.description = Unquote(value); + } else if (key == "depends_on") { + auto deps = ParseInlineList(value); + group.depends_on.insert(group.depends_on.end(), deps.begin(), deps.end()); + } else { + return absl::InvalidArgumentError( + absl::StrCat("Unknown field in group entry: '", key, "'")); + } + } + + ++(*index); + + while (*index < lines.size()) { + std::string raw = StripComment(lines[*index]); + std::string trimmed = Trim(raw); + int indent = CountIndent(raw); + if (trimmed.empty()) { + ++(*index); + continue; + } + if (indent <= base_indent) { + break; + } + if (indent == base_indent + 2) { + std::string key; + std::string value; + if (!ParseKeyValue(trimmed, &key, &value)) { + return absl::InvalidArgumentError( + "Expected key/value pair inside group definition"); + } + if (key == "name") { + group.name = Unquote(value); + ++(*index); + } else if (key == "description") { + group.description = Unquote(value); + ++(*index); + } else if (key == "depends_on") { + if (!value.empty()) { + auto deps = ParseInlineList(value); + group.depends_on.insert(group.depends_on.end(), deps.begin(), + deps.end()); + ++(*index); + } else { + ++(*index); + RETURN_IF_ERROR(ParseStringListBlock(lines, index, base_indent + 4, + &group.depends_on)); + } + } else if (key == "tests") { + if (!value.empty()) { + return absl::InvalidArgumentError( + "tests block must be defined as indented list"); + } + ++(*index); + RETURN_IF_ERROR( + ParseTestsBlock(lines, index, base_indent + 4, &group)); + } else { + return absl::InvalidArgumentError( + absl::StrCat("Unknown attribute in group definition: '", key, + "'")); + } + } else { + return absl::InvalidArgumentError( + "Unexpected indentation inside group definition"); + } + } + + if (group.name.empty()) { + return absl::InvalidArgumentError( + "Each test group must define a name"); + } + suite->groups.push_back(std::move(group)); + return absl::OkStatus(); +} + +absl::Status ParseGroupBlock(const std::vector& lines, + size_t* index, TestSuiteDefinition* suite) { + while (*index < lines.size()) { + std::string raw = StripComment(lines[*index]); + std::string trimmed = Trim(raw); + int indent = CountIndent(raw); + if (trimmed.empty()) { + ++(*index); + continue; + } + if (indent < 2) { + break; + } + if (indent != 2) { + return absl::InvalidArgumentError( + "Invalid indentation inside test_groups block"); + } + RETURN_IF_ERROR(ParseGroupEntry(lines, index, suite)); + } + return absl::OkStatus(); +} + +absl::Status ParseConfigBlock(const std::vector& lines, + size_t* index, TestSuiteConfig* config) { + while (*index < lines.size()) { + std::string raw = StripComment(lines[*index]); + std::string trimmed = Trim(raw); + int indent = CountIndent(raw); + if (trimmed.empty()) { + ++(*index); + continue; + } + if (indent < 2) { + break; + } + if (indent != 2) { + return absl::InvalidArgumentError( + "Invalid indentation inside config block"); + } + std::string key; + std::string value; + if (!ParseKeyValue(trimmed, &key, &value)) { + return absl::InvalidArgumentError( + "Expected key/value pair inside config block"); + } + RETURN_IF_ERROR(ParseScalarConfig(key, value, config)); + ++(*index); + } + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr ParseTestSuiteDefinition( + absl::string_view content) { + std::vector lines = absl::StrSplit(content, '\n'); + TestSuiteDefinition suite; + size_t index = 0; + + while (index < lines.size()) { + std::string raw = StripComment(lines[index]); + std::string trimmed = Trim(raw); + if (trimmed.empty()) { + ++index; + continue; + } + + int indent = CountIndent(raw); + if (indent != 0) { + return absl::InvalidArgumentError( + "Top-level entries must not be indented in suite definition"); + } + + std::string key; + std::string value; + if (!ParseKeyValue(trimmed, &key, &value)) { + return absl::InvalidArgumentError( + absl::StrCat("Malformed top-level entry: '", trimmed, "'")); + } + + if (key == "name") { + suite.name = Unquote(value); + ++index; + } else if (key == "description") { + suite.description = Unquote(value); + ++index; + } else if (key == "version") { + suite.version = Unquote(value); + ++index; + } else if (key == "config") { + if (!value.empty()) { + return absl::InvalidArgumentError( + "config block must not specify inline value"); + } + ++index; + RETURN_IF_ERROR(ParseConfigBlock(lines, &index, &suite.config)); + } else if (key == "test_groups") { + if (!value.empty()) { + return absl::InvalidArgumentError( + "test_groups must be defined as an indented list"); + } + ++index; + RETURN_IF_ERROR(ParseGroupBlock(lines, &index, &suite)); + } else { + return absl::InvalidArgumentError( + absl::StrCat("Unknown top-level key: '", key, "'")); + } + } + + if (suite.name.empty()) { + suite.name = "Unnamed Suite"; + } + if (suite.version.empty()) { + suite.version = "1.0"; + } + return suite; +} + +absl::StatusOr LoadTestSuiteFromFile( + const std::string& path) { + std::ifstream file(path); + if (!file.is_open()) { + return absl::NotFoundError( + absl::StrCat("Failed to open test suite file '", path, "'")); + } + std::stringstream buffer; + buffer << file.rdbuf(); + return ParseTestSuiteDefinition(buffer.str()); +} + +} // namespace cli +} // namespace yaze diff --git a/src/cli/service/test_suite_loader.h b/src/cli/service/test_suite_loader.h new file mode 100644 index 00000000..ca64d974 --- /dev/null +++ b/src/cli/service/test_suite_loader.h @@ -0,0 +1,19 @@ +#ifndef YAZE_CLI_SERVICE_TEST_SUITE_LOADER_H_ +#define YAZE_CLI_SERVICE_TEST_SUITE_LOADER_H_ + +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "cli/service/test_suite.h" + +namespace yaze { +namespace cli { + +absl::StatusOr ParseTestSuiteDefinition(absl::string_view content); +absl::StatusOr LoadTestSuiteFromFile(const std::string& path); + +} // namespace cli +} // namespace yaze + +#endif // YAZE_CLI_SERVICE_TEST_SUITE_LOADER_H_ diff --git a/src/cli/service/test_suite_reporter.cc b/src/cli/service/test_suite_reporter.cc new file mode 100644 index 00000000..c99e3ad2 --- /dev/null +++ b/src/cli/service/test_suite_reporter.cc @@ -0,0 +1,293 @@ +#include "cli/service/test_suite_reporter.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" +#include "util/macro.h" + +namespace yaze { +namespace cli { +namespace { + +std::string OutcomeToString(TestCaseOutcome outcome) { + switch (outcome) { + case TestCaseOutcome::kPassed: + return "PASS"; + case TestCaseOutcome::kFailed: + return "FAIL"; + case TestCaseOutcome::kError: + return "ERROR"; + case TestCaseOutcome::kSkipped: + return "SKIP"; + } + return "UNKNOWN"; +} + +std::string EscapeXml(absl::string_view input) { + std::string escaped; + escaped.reserve(input.size()); + for (char c : input) { + switch (c) { + case '&': + escaped.append("&"); + break; + case '<': + escaped.append("<"); + break; + case '>': + escaped.append(">"); + break; + case '\"': + escaped.append("""); + break; + case '\'': + escaped.append("'"); + break; + default: + escaped.push_back(c); + break; + } + } + return escaped; +} + +std::string JoinLogs(const std::vector& logs) { + if (logs.empty()) { + return {}; + } + return absl::StrJoin(logs, "\n"); +} + +void ComputeSummaryCounters(const TestSuiteRunSummary& summary, int* total, + int* passed, int* failed, int* errors, + int* skipped) { + *total = static_cast(summary.results.size()); + *passed = summary.passed; + *failed = summary.failed; + *errors = summary.errors; + *skipped = summary.skipped; + + if (*passed + *failed + *errors + *skipped != *total) { + *passed = *failed = *errors = *skipped = 0; + for (const auto& result : summary.results) { + switch (result.outcome) { + case TestCaseOutcome::kPassed: + ++(*passed); + break; + case TestCaseOutcome::kFailed: + ++(*failed); + break; + case TestCaseOutcome::kError: + ++(*errors); + break; + case TestCaseOutcome::kSkipped: + ++(*skipped); + break; + } + } + } +} + +absl::Duration TotalDuration(const TestSuiteRunSummary& summary) { + if (summary.total_duration > absl::ZeroDuration()) { + return summary.total_duration; + } + absl::Duration total = absl::ZeroDuration(); + for (const auto& result : summary.results) { + total += result.duration; + } + return total; +} + +} // namespace + +std::string BuildTextSummary(const TestSuiteRunSummary& summary) { + std::ostringstream oss; + int total = 0; + int passed = 0; + int failed = 0; + int errors = 0; + int skipped = 0; + ComputeSummaryCounters(summary, &total, &passed, &failed, &errors, &skipped); + + absl::Time timestamp = summary.started_at; + if (timestamp == absl::InfinitePast()) { + timestamp = absl::Now(); + } + + oss << "Suite: " + << (summary.suite && !summary.suite->name.empty() ? summary.suite->name + : "Unnamed Suite") + << "\n"; + oss << "Started: " + << absl::FormatTime("%Y-%m-%d %H:%M:%S", timestamp, absl::UTCTimeZone()) + << " UTC\n"; + oss << "Totals: " << total << " (" << passed << " passed, " << failed + << " failed, " << errors << " errors, " << skipped << " skipped)"; + + absl::Duration duration = TotalDuration(summary); + if (duration > absl::ZeroDuration()) { + oss << " in " + << absl::StrFormat("%.2fs", absl::ToDoubleSeconds(duration)); + } + oss << "\n\n"; + + for (const auto& result : summary.results) { + std::string group_name; + if (result.group) { + group_name = result.group->name; + } else if (result.test) { + group_name = result.test->group_name; + } + std::string test_name = result.test ? result.test->name : ""; + oss << " [" << OutcomeToString(result.outcome) << "] "; + if (!group_name.empty()) { + oss << group_name << " :: "; + } + oss << test_name; + if (result.duration > absl::ZeroDuration()) { + oss << " (" << absl::StrFormat("%.2fs", + absl::ToDoubleSeconds(result.duration)) + << ")"; + } + oss << "\n"; + if (!result.message.empty() && + result.outcome != TestCaseOutcome::kPassed) { + oss << " " << result.message << "\n"; + } + if (!result.assertions.empty() && + result.outcome != TestCaseOutcome::kPassed) { + for (const auto& assertion : result.assertions) { + oss << " - " << assertion.description << " : " + << (assertion.passed ? "PASS" : "FAIL"); + if (!assertion.error_message.empty()) { + oss << " (" << assertion.error_message << ")"; + } + oss << "\n"; + } + } + } + + return oss.str(); +} + +absl::StatusOr BuildJUnitReport( + const TestSuiteRunSummary& summary) { + std::ostringstream oss; + int total = 0; + int passed = 0; + int failed = 0; + int errors = 0; + int skipped = 0; + ComputeSummaryCounters(summary, &total, &passed, &failed, &errors, &skipped); + + absl::Time timestamp = summary.started_at; + if (timestamp == absl::InfinitePast()) { + timestamp = absl::Now(); + } + absl::Duration duration = TotalDuration(summary); + + std::string suite_name = + summary.suite ? summary.suite->name : "YAZE GUI Test Suite"; + + oss << "\n"; + oss << "\n"; + + for (const auto& result : summary.results) { + std::string classname; + if (result.group) { + classname = result.group->name; + } else if (result.test) { + classname = result.test->group_name; + } + std::string test_name = result.test ? result.test->name : "Test"; + oss << " "; + + if (result.outcome == TestCaseOutcome::kFailed) { + std::string body = result.message; + if (!result.assertions.empty()) { + std::vector assertion_lines; + for (const auto& assertion : result.assertions) { + assertion_lines.push_back( + absl::StrCat(assertion.description, " => ", + assertion.passed ? "PASS" : "FAIL")); + } + body = absl::StrCat(body, "\n", + absl::StrJoin(assertion_lines, "\n")); + } + oss << "\n " << EscapeXml(body) << ""; + if (!result.logs.empty()) { + oss << "\n " << EscapeXml(JoinLogs(result.logs)) + << ""; + } + oss << "\n \n"; + continue; + } + + if (result.outcome == TestCaseOutcome::kError) { + std::string detail = result.message; + if (!result.logs.empty()) { + detail = absl::StrCat(detail, "\n", JoinLogs(result.logs)); + } + oss << "\n " << EscapeXml(detail) << ""; + oss << "\n \n"; + continue; + } + + if (!result.logs.empty()) { + oss << "\n " << EscapeXml(JoinLogs(result.logs)) + << ""; + } + oss << "\n \n"; + } + + oss << "\n"; + return oss.str(); +} + +absl::Status WriteJUnitReport(const TestSuiteRunSummary& summary, + const std::string& output_path) { + ASSIGN_OR_RETURN(std::string xml, BuildJUnitReport(summary)); + std::filesystem::path path(output_path); + if (path.has_parent_path()) { + std::error_code ec; + std::filesystem::create_directories(path.parent_path(), ec); + if (ec) { + return absl::InternalError( + absl::StrCat("Failed to create directory for JUnit report: ", + ec.message())); + } + } + std::ofstream out(path); + if (!out.is_open()) { + return absl::InternalError( + absl::StrCat("Unable to open JUnit output file '", output_path, + "'")); + } + out << xml; + return absl::OkStatus(); +} + +} // namespace cli +} // namespace yaze diff --git a/src/cli/service/test_suite_reporter.h b/src/cli/service/test_suite_reporter.h new file mode 100644 index 00000000..3dfe52d9 --- /dev/null +++ b/src/cli/service/test_suite_reporter.h @@ -0,0 +1,20 @@ +#ifndef YAZE_CLI_SERVICE_TEST_SUITE_REPORTER_H_ +#define YAZE_CLI_SERVICE_TEST_SUITE_REPORTER_H_ + +#include + +#include "absl/status/statusor.h" +#include "cli/service/test_suite.h" + +namespace yaze { +namespace cli { + +std::string BuildTextSummary(const TestSuiteRunSummary& summary); +absl::StatusOr BuildJUnitReport(const TestSuiteRunSummary& summary); +absl::Status WriteJUnitReport(const TestSuiteRunSummary& summary, + const std::string& output_path); + +} // namespace cli +} // namespace yaze + +#endif // YAZE_CLI_SERVICE_TEST_SUITE_REPORTER_H_