From 287f04ffc420edc9d74ecc803d4c5709aeb20072 Mon Sep 17 00:00:00 2001 From: scawful Date: Fri, 3 Oct 2025 00:39:02 -0400 Subject: [PATCH] feat: Implement CLI test suite commands with YAML support and interactive creation --- docs/z3ed/E6-z3ed-cli-design.md | 25 +- docs/z3ed/E6-z3ed-implementation-plan.md | 53 ++-- docs/z3ed/E6-z3ed-reference.md | 13 +- docs/z3ed/README.md | 6 +- src/cli/handlers/agent/test_commands.cc | 334 ++++++++++++++++++++++- src/cli/modern_cli.cc | 1 + src/cli/service/test_suite_writer.cc | 181 ++++++++++++ src/cli/service/test_suite_writer.h | 26 ++ src/cli/z3ed.cmake | 2 + 9 files changed, 610 insertions(+), 31 deletions(-) create mode 100644 src/cli/service/test_suite_writer.cc create mode 100644 src/cli/service/test_suite_writer.h diff --git a/docs/z3ed/E6-z3ed-cli-design.md b/docs/z3ed/E6-z3ed-cli-design.md index e5d2b13e..b423a0da 100644 --- a/docs/z3ed/E6-z3ed-cli-design.md +++ b/docs/z3ed/E6-z3ed-cli-design.md @@ -335,14 +335,27 @@ z3ed_client.Click(target="button:Save Changes") - EditorManager gains on-screen diagnostics tied to harness artifacts - Lays groundwork for future telemetry and CI reporting -#### IT-09: CI/CD Integration (2-3 hours) +#### IT-09: CI/CD Integration ✅ CLI Foundations Complete **Problem**: Tests run manually. No automated regression on PR/merge. -**Solution**: Standardize test execution for CI: -- YAML test suite format (groups, dependencies, parallel execution) -- `z3ed test suite run` command with `--ci-mode` -- JUnit XML output for CI parsers (Jenkins, GitHub Actions) -- Exit codes: 0=pass, 1=fail, 2=error +**Shipped**: +- YAML test suite runtime with dependency-aware execution and retry handling +- `z3ed agent test suite run` supports `--group`, `--tag`, `--param`, + `--retries`, `--ci-mode`, and automatic JUnit XML emission under + `test-results/junit/` +- `z3ed agent test suite validate` performs structural linting and surfaces + exit codes (0 pass, 1 fail, 2 error) +- NEW `z3ed agent test suite create` interactive builder generates suites + (defaulting to `tests/.yaml`), with prompts for groups, replay scripts, + tags, and key=value parameters. `--force` enables overwrite flows. + +**Next Integration Steps**: +- Publish canonical `tests/smoke.yaml` / `tests/regression.yaml` templates in + the repo +- Add GitHub Actions example wiring harness referencing the new runner +- Document best practices for mapping suite tags to CI stages (smoke, + regression, nightly) +- Wire run summaries into docs (`docs/testing/`) with badge-ready status tables **GitHub Actions Example**: ```yaml diff --git a/docs/z3ed/E6-z3ed-implementation-plan.md b/docs/z3ed/E6-z3ed-implementation-plan.md index f28a0146..07dbddce 100644 --- a/docs/z3ed/E6-z3ed-implementation-plan.md +++ b/docs/z3ed/E6-z3ed-implementation-plan.md @@ -31,6 +31,12 @@ The z3ed CLI and AI agent workflow system has completed major infrastructure mil - IT-08c: Widget state dumps with comprehensive UI snapshot (JSON, 45 min) - Proto schema updated with screenshot_path, failure_context, widget_state - GetTestResults RPC returns complete failure diagnostics +- **✅ IT-09 CLI Suite Commands Landed**: End-to-end suite orchestration for CI + - `agent test suite run` handles groups, tags, params, retries, and emits + summaries plus default JUnit XML under `test-results/junit/` + - `agent test suite validate` performs structural linting with exit codes + - NEW `agent test suite create` interactive builder writes YAML suites to + `tests/.yaml` (with `--force` overwrite) and guides group/test entry - **✅ IT-08a Screenshot RPC Complete**: SDL-based screenshot capture operational - Captures 1536x864 BMP files via SDL_RenderReadPixels - Successfully tested via gRPC (5.3MB output files) @@ -315,23 +321,30 @@ message DiscoverWidgetsResponse { } ``` -#### IT-09: CI/CD Integration (2-3 hours) -**Implementation Tasks**: -1. **Standardized Test Suite Format**: - - YAML/JSON format for test suite definitions - - Support test groups (smoke, regression, nightly) - - Enable parallel execution with dependencies - -2. **CI-Friendly CLI**: - - `z3ed test run-suite tests/suite.yaml --ci-mode` - - Exit codes: 0 = all passed, 1 = failures, 2 = errors - - JUnit XML output for CI parsers - - GitHub Actions integration examples - -3. **Documentation**: - - Add `.github/workflows/gui-tests.yml` example - - Create sample test suites for common scenarios - - Document best practices for flaky test handling +#### IT-09: CI/CD Integration ✅ CLI Tooling Shipped +**Delivered (Oct 3, 2025)**: +1. **Standardized Suite Runtime** + - YAML suite parser/loader with group dependencies and retry semantics + - `z3ed agent test suite run` exposes `--group`, `--tag`, `--param`, + `--retries`, `--ci-mode`, and `--junit` + - Automatic JUnit XML emission to `test-results/junit/.xml` + +2. **Validation & Authoring UX** + - `z3ed agent test suite validate` surfaces structural linting with + annotated exit codes (0 pass, 1 fail, 2 error) + - NEW `z3ed agent test suite create ` interactive flow scaffolds + suites under `tests/`, prompting for metadata, groups, replay scripts, + tags, and key=value parameters (with `--force` overwrite support) + +3. **Reporting** + - Text and JSON summaries include per-test assertions and retry outcomes + - Default output directory layout ready for CI artifact upload + +**Next Steps** (post-CLI follow-through): +- Publish canonical `tests/smoke.yaml` / `tests/regression.yaml` samples +- Add `.github/workflows/gui-tests.yml` template referencing the new runner +- Document flaky-test mitigation patterns, including recommended retry counts +- Wire suite execution output into docs/CI dashboards for quick triage **Test Suite Format**: ```yaml @@ -541,11 +554,11 @@ z3ed collab replay session_2025_10_02.yaml --speed 2x | IT-05 | Add test introspection RPCs (GetTestStatus, ListTests, GetResults) | ImGuiTest Bridge | Code | ✅ Done | IT-01 - Enable clients to poll test results and query execution state (Oct 2, 2025) | | IT-06 | Implement widget discovery API for AI agents | ImGuiTest Bridge | Code | 📋 Planned | IT-01 - DiscoverWidgets RPC to enumerate windows, buttons, inputs | | IT-07 | Add test recording/replay for regression testing | ImGuiTest Bridge | Code | ✅ Done | IT-05 - RecordSession/ReplaySession RPCs with JSON test scripts | -| IT-08 | Enhance error reporting with screenshots and state dumps | ImGuiTest Bridge | Code | 🔄 Active | IT-01 - Capture widget state on failure for debugging (67% complete: IT-08a ✅, IT-08b ✅, IT-08c 🔄) | +| IT-08 | Enhance error reporting with screenshots and state dumps | ImGuiTest Bridge | Code | ✅ Done | IT-01 - Screenshot RPC, auto-capture, widget state dumps complete (Oct 2, 2025) | | IT-08a | Screenshot RPC implementation (SDL capture) | ImGuiTest Bridge | Code | ✅ Done | IT-01 - Screenshot capture complete (Oct 2, 2025) | | IT-08b | Auto-capture screenshots on test failure | ImGuiTest Bridge | Code | ✅ Done | IT-08a - Integrated with TestManager (Oct 2, 2025) | -| IT-08c | Widget state dumps and execution context | ImGuiTest Bridge | Code | � Active | IT-08b - Enhanced failure diagnostics (NEXT PRIORITY) | -| IT-09 | Create standardized test suite format for CI integration | ImGuiTest Bridge | Infra | 📋 Planned | IT-07 - JSON/YAML test suite format compatible with CI/CD pipelines | +| IT-08c | Widget state dumps and execution context | ImGuiTest Bridge | Code | ✅ Done | IT-08b - Enhanced failure diagnostics (Oct 2, 2025) | +| IT-09 | Create standardized test suite format for CI integration | ImGuiTest Bridge | Infra | ✅ Done | IT-07 - CLI suite run/validate/create commands, JUnit output | | IT-10 | Collaborative editing & multiplayer sessions with shared AI | Collaboration | Feature | 📋 Planned | IT-05, IT-08 - Real-time multi-user editing with live cursors, shared proposals (12-15 hours) | | VP-01 | Expand CLI unit tests for new commands and sandbox flow. | Verification Pipeline | Test | 📋 Planned | RC/AW tasks | | VP-02 | Add harness integration tests with replay scripts. | Verification Pipeline | Test | 📋 Planned | IT tasks | diff --git a/docs/z3ed/E6-z3ed-reference.md b/docs/z3ed/E6-z3ed-reference.md index 14590fd4..34c114fb 100644 --- a/docs/z3ed/E6-z3ed-reference.md +++ b/docs/z3ed/E6-z3ed-reference.md @@ -494,12 +494,23 @@ z3ed agent test suite [options] Actions: run Run test suite (YAML/JSON) - create Create new test suite interactively + create Create new test suite interactively (writes tests/.yaml) validate Validate test suite format +Options for `create`: + --force Overwrite the target file without prompting + +The create workflow walks you through suite metadata, groups, and tests. If +you pass a bare name (e.g., `smoke`), the suite is written to +`tests/smoke.yaml`; supplying a path like `ci/regression.yaml` preserves the +directory. Use the prompts to add groups, associate JSON replay scripts, tags, +and key=value parameters. The CLI warns if referenced scripts are missing but +still records them so you can author them later. + Examples: z3ed agent test suite run tests/smoke.yaml z3ed agent test suite validate tests/regression.yaml + z3ed agent test suite create smoke ``` ### ROM Commands diff --git a/docs/z3ed/README.md b/docs/z3ed/README.md index f52f9c36..04153444 100644 --- a/docs/z3ed/README.md +++ b/docs/z3ed/README.md @@ -81,7 +81,11 @@ See the **[Technical Reference](E6-z3ed-reference.md)** for a full command list. ## Recent Enhancements -**Recent Progress (Oct 2, 2025)** +**Recent Progress (Oct 3, 2025)** +- ✅ IT-09 CLI Test Suite Tooling Complete: run/validate/create commands + JUnit output + - Full suite runner with group/tag filters, parametrization, retries, and CI-friendly exit codes + - Interactive `agent test suite create` scaffolds YAML definitions in `tests/` + - Default JUnit reports under `test-results/junit/` for CI upload - ✅ IT-08 Enhanced Error Reporting Complete: Full diagnostic capture on test failures - IT-08a: Screenshot RPC with SDL capture (BMP format, 1536x864) - IT-08b: Auto-capture execution context on failures (frame, window, widget) diff --git a/src/cli/handlers/agent/test_commands.cc b/src/cli/handlers/agent/test_commands.cc index bbd5680a..74c3f4e5 100644 --- a/src/cli/handlers/agent/test_commands.cc +++ b/src/cli/handlers/agent/test_commands.cc @@ -3,10 +3,13 @@ #include #include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -36,9 +39,16 @@ #include "cli/service/test_suite.h" #include "cli/service/test_suite_loader.h" #include "cli/service/test_suite_reporter.h" +#include "cli/service/test_suite_writer.h" #include "cli/service/test_workflow_generator.h" #include "util/macro.h" +#if defined(_WIN32) +#include +#else +#include +#endif + namespace yaze { namespace cli { namespace agent { @@ -55,6 +65,126 @@ void AttachExitCode(absl::Status* status, int exit_code) { absl::Cord(std::to_string(exit_code))); } +std::string TrimWhitespace(absl::string_view value) { + return std::string(absl::StripAsciiWhitespace(value)); +} + +bool IsInteractiveInput() { +#if defined(_WIN32) + return _isatty(_fileno(stdin)) != 0; +#else + return isatty(fileno(stdin)) != 0; +#endif +} + +std::string PromptWithDefault(const std::string& prompt, + const std::string& default_value, + bool allow_empty = true) { + while (true) { + std::cout << prompt; + if (!default_value.empty()) { + std::cout << " [" << default_value << "]"; + } + std::cout << ": "; + std::cout.flush(); + + std::string line; + if (!std::getline(std::cin, line)) { + return default_value; + } + std::string trimmed = TrimWhitespace(line); + if (!trimmed.empty()) { + return trimmed; + } + if (!default_value.empty()) { + return default_value; + } + if (allow_empty) { + return std::string(); + } + std::cout << " Value is required." << std::endl; + } +} + +std::string PromptRequired(const std::string& prompt, + const std::string& default_value = std::string()) { + return PromptWithDefault(prompt, default_value, /*allow_empty=*/false); +} + +int PromptInt(const std::string& prompt, int default_value, int min_value) { + while (true) { + std::string default_str = absl::StrCat(default_value); + std::string input = PromptWithDefault(prompt, default_str); + if (input.empty()) { + return default_value; + } + int value = 0; + if (absl::SimpleAtoi(input, &value) && value >= min_value) { + return value; + } + std::cout << " Enter an integer >= " << min_value << "." << std::endl; + } +} + +bool PromptYesNo(const std::string& prompt, bool default_value) { + while (true) { + std::cout << prompt << " [" << (default_value ? "Y/n" : "y/N") + << "]: "; + std::cout.flush(); + std::string line; + if (!std::getline(std::cin, line)) { + return default_value; + } + std::string trimmed = TrimWhitespace(line); + if (trimmed.empty()) { + return default_value; + } + char c = static_cast(std::tolower(static_cast(trimmed[0]))); + if (c == 'y') { + return true; + } + if (c == 'n') { + return false; + } + std::cout << " Please respond with 'y' or 'n'." << std::endl; + } +} + +std::vector ParseCommaSeparated(absl::string_view input) { + std::vector values; + for (absl::string_view token : absl::StrSplit(input, ',')) { + std::string trimmed = TrimWhitespace(token); + if (!trimmed.empty()) { + values.push_back(trimmed); + } + } + return values; +} + +bool ParseKeyValueEntry(const std::string& input, std::string* key, + std::string* value) { + size_t equals = input.find('='); + if (equals == std::string::npos) { + return false; + } + *key = TrimWhitespace(absl::string_view(input.data(), equals)); + *value = TrimWhitespace(absl::string_view(input.data() + equals + 1, + input.size() - equals - 1)); + return !key->empty(); +} + +std::string DeriveTestNameFromPath(const std::string& path) { + if (path.empty()) { + return ""; + } + std::filesystem::path fs_path(path); + std::string stem = fs_path.stem().string(); + if (!stem.empty()) { + return stem; + } + return path; +} + std::string OutcomeToLabel(TestCaseOutcome outcome) { switch (outcome) { case TestCaseOutcome::kPassed: @@ -1967,10 +2097,209 @@ absl::Status HandleTestSuiteValidateCommand( return absl::OkStatus(); } +absl::Status HandleTestSuiteCreateCommand( + const std::vector& arg_vec) { + if (!IsInteractiveInput()) { + return absl::FailedPreconditionError( + "agent test suite create requires an interactive terminal"); + } + + std::string target_arg; + bool force = false; + for (size_t i = 0; i < arg_vec.size(); ++i) { + const std::string& token = arg_vec[i]; + if (token == "--force") { + force = true; + continue; + } + if (absl::StartsWith(token, "--force=")) { + std::string value = TrimWhitespace(token.substr(8)); + absl::AsciiStrToLower(&value); + force = (value == "1" || value == "true" || value == "yes" || + value == "on"); + continue; + } + if (absl::StartsWith(token, "--")) { + return absl::InvalidArgumentError( + absl::StrCat("Unknown flag for agent test suite create: ", token)); + } + if (!target_arg.empty()) { + return absl::InvalidArgumentError( + "agent test suite create accepts a single or "); + } + target_arg = token; + } + + if (target_arg.empty()) { + return absl::InvalidArgumentError( + "Usage: agent test suite create [--force]"); + } + + std::filesystem::path output_path(target_arg); + bool looks_like_path = target_arg.find('/') != std::string::npos || + target_arg.find('\\') != std::string::npos || + target_arg.find('.') != std::string::npos; + if (!looks_like_path) { + output_path = std::filesystem::path("tests") / + std::filesystem::path(target_arg + ".yaml"); + } else if (output_path.extension().empty()) { + output_path.replace_extension(".yaml"); + } + + std::string extension = output_path.extension().string(); + absl::AsciiStrToLower(&extension); + if (!extension.empty() && extension != ".yaml" && extension != ".yml") { + return absl::InvalidArgumentError( + "Only .yaml/.yml suites are supported."); + } + if (extension == ".yml") { + output_path.replace_extension(".yaml"); + } + + std::string default_suite_name = output_path.stem().string(); + if (default_suite_name.empty()) { + default_suite_name = "New Suite"; + } + + std::error_code exists_ec; + if (!force && std::filesystem::exists(output_path, exists_ec) && !exists_ec) { + std::string question = + absl::StrCat("File ", output_path.string(), + " already exists. Overwrite?"); + if (!PromptYesNo(question, false)) { + return absl::CancelledError("Suite creation cancelled by user"); + } + force = true; + } + + std::cout << "=== Test Suite Metadata ===" << std::endl; + TestSuiteDefinition suite; + suite.name = TrimWhitespace(PromptRequired("Suite name", default_suite_name)); + if (suite.name.empty()) { + suite.name = default_suite_name; + } + suite.description = TrimWhitespace( + PromptWithDefault("Suite description", std::string())); + suite.version = TrimWhitespace(PromptWithDefault("Suite version", "1.0")); + if (suite.version.empty()) { + suite.version = "1.0"; + } + suite.config.timeout_seconds = + PromptInt("Timeout per test (seconds)", 30, 0); + suite.config.retry_on_failure = + PromptInt("Retries per test", 0, 0); + suite.config.parallel_execution = + PromptYesNo("Enable parallel execution?", false); + + std::cout << "\n=== Define Test Groups ===" << std::endl; + while (true) { + std::string group_name = TrimWhitespace( + PromptWithDefault("Add group name (leave blank to finish)", + std::string())); + if (group_name.empty()) { + break; + } + + TestGroupDefinition group; + group.name = group_name; + group.description = TrimWhitespace( + PromptWithDefault(" Group description", std::string())); + std::string deps_input = TrimWhitespace( + PromptWithDefault(" Depends on (comma separated)", + std::string())); + group.depends_on = ParseCommaSeparated(deps_input); + + std::cout << " Adding tests for group '" << group.name << "'" << std::endl; + while (true) { + std::string script_prompt = + absl::StrCat(" Test script path (JSON) [blank to finish group] "); + std::string script_path = TrimWhitespace( + PromptWithDefault(script_prompt, std::string())); + if (script_path.empty()) { + break; + } + + TestCaseDefinition test; + test.group_name = group.name; + test.script_path = script_path; + + std::string default_test_name = DeriveTestNameFromPath(script_path); + std::string name_input = TrimWhitespace( + PromptWithDefault(" Display name", default_test_name)); + test.name = name_input.empty() ? default_test_name : name_input; + test.description = TrimWhitespace( + PromptWithDefault(" Test description", std::string())); + std::string tags_input = TrimWhitespace( + PromptWithDefault(" Tags (comma separated)", std::string())); + test.tags = ParseCommaSeparated(tags_input); + + while (true) { + std::string param_input = TrimWhitespace(PromptWithDefault( + " Parameter key=value (blank to finish)", std::string())); + if (param_input.empty()) { + break; + } + std::string key; + std::string value; + if (!ParseKeyValueEntry(param_input, &key, &value)) { + std::cout << " Expected key=value" << std::endl; + continue; + } + test.parameters[key] = value; + } + + if (test.id.empty()) { + test.id = absl::StrCat(group.name, ":", test.name); + } + + std::error_code file_check_ec; + if (!std::filesystem::exists(script_path, file_check_ec) || file_check_ec) { + std::cout << " (warning: file not found)" << std::endl; + } + + group.tests.push_back(std::move(test)); + std::cout << std::endl; + } + + if (group.tests.empty()) { + if (!PromptYesNo(" No tests added. Keep empty group?", false)) { + continue; + } + } + + suite.groups.push_back(std::move(group)); + std::cout << std::endl; + } + + if (suite.groups.empty()) { + if (!PromptYesNo("No groups defined. Create empty suite anyway?", false)) { + return absl::CancelledError("Suite creation cancelled"); + } + } + + int total_tests = 0; + for (const auto& group : suite.groups) { + total_tests += static_cast(group.tests.size()); + } + + absl::Status write_status = + WriteTestSuiteToFile(suite, output_path.string(), force); + if (!write_status.ok()) { + return write_status; + } + + std::cout << "\nCreated suite '" << suite.name << "' at " + << output_path.string() << "\n"; + std::cout << " Groups: " << suite.groups.size() << "\n"; + std::cout << " Tests: " << total_tests << "\n"; + + return absl::OkStatus(); +} + absl::Status HandleTestSuiteCommand(const std::vector& arg_vec) { if (arg_vec.empty()) { return absl::InvalidArgumentError( - "Usage: agent test suite [options]"); + "Usage: agent test suite [options]"); } const std::string& action = arg_vec[0]; @@ -1983,8 +2312,7 @@ absl::Status HandleTestSuiteCommand(const std::vector& arg_vec) { return HandleTestSuiteValidateCommand(tail); } if (action == "create") { - return absl::UnimplementedError( - "agent test suite create is not implemented yet"); + return HandleTestSuiteCreateCommand(tail); } return absl::InvalidArgumentError( diff --git a/src/cli/modern_cli.cc b/src/cli/modern_cli.cc index 4c8795f0..c21c4d61 100644 --- a/src/cli/modern_cli.cc +++ b/src/cli/modern_cli.cc @@ -65,6 +65,7 @@ void ModernCLI::SetupCommands() { " test status: status --test-id [--follow] [--host ] [--port ]\n" " test list: list [--category ] [--status ] [--limit ] [--host ] [--port ]\n" " test results: results --test-id [--include-logs] [--format yaml|json] [--host ] [--port ]\n" + " test suite: suite [options]\n" " gui discover: discover [--window ] [--type ] [--path-prefix ]\n" " [--include-invisible] [--include-disabled] [--format table|json] [--limit ]\n" " describe options: [--resource ] [--format json|yaml] [--output ]\n" diff --git a/src/cli/service/test_suite_writer.cc b/src/cli/service/test_suite_writer.cc new file mode 100644 index 00000000..402ebb23 --- /dev/null +++ b/src/cli/service/test_suite_writer.cc @@ -0,0 +1,181 @@ +#include "cli/service/test_suite_writer.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" + +namespace yaze { +namespace cli { +namespace { + +std::string Indent(int count) { return std::string(count, ' '); } + +std::string QuoteYaml(absl::string_view value) { + std::string escaped(value); + absl::StrReplaceAll({{"\\", "\\\\"}, {"\"", "\\\""}}, &escaped); + return absl::StrCat("\"", escaped, "\""); +} + +void AppendLine(std::string* out, int indent, absl::string_view line) { + out->append(Indent(indent)); + out->append(line.data(), line.size()); + out->append("\n"); +} + +void AppendScalar(std::string* out, int indent, absl::string_view key, + absl::string_view value, bool quote) { + out->append(Indent(indent)); + out->append(key.data(), key.size()); + out->append(":"); + if (!value.empty()) { + out->append(" "); + if (quote) { + out->append(QuoteYaml(value)); + } else { + out->append(value.data(), value.size()); + } + } + out->append("\n"); +} + +std::string FormatDuration(int seconds) { + if (seconds <= 0) { + return "0s"; + } + if (seconds % 60 == 0) { + return absl::StrCat(seconds / 60, "m"); + } + return absl::StrCat(seconds, "s"); +} + +std::string FormatBool(bool value) { return value ? "true" : "false"; } + +std::string JoinQuotedList(const std::vector& values) { + if (values.empty()) { + return "[]"; + } + std::vector quoted; + quoted.reserve(values.size()); + for (const auto& v : values) { + quoted.push_back(QuoteYaml(v)); + } + return absl::StrCat("[", absl::StrJoin(quoted, ", "), "]"); +} + +} // namespace + +std::string BuildTestSuiteYaml(const TestSuiteDefinition& suite) { + std::string output; + + if (!suite.name.empty()) { + AppendScalar(&output, 0, "name", suite.name, /*quote=*/true); + } else { + AppendScalar(&output, 0, "name", "Unnamed Suite", /*quote=*/true); + } + if (!suite.description.empty()) { + AppendScalar(&output, 0, "description", suite.description, + /*quote=*/true); + } + if (!suite.version.empty()) { + AppendScalar(&output, 0, "version", suite.version, /*quote=*/true); + } + + AppendLine(&output, 0, "config:"); + AppendScalar(&output, 2, "timeout_per_test", + FormatDuration(suite.config.timeout_seconds), + /*quote=*/false); + AppendScalar(&output, 2, "retry_on_failure", + absl::StrCat(suite.config.retry_on_failure), + /*quote=*/false); + AppendScalar(&output, 2, "parallel_execution", + FormatBool(suite.config.parallel_execution), + /*quote=*/false); + + AppendLine(&output, 0, "test_groups:"); + for (size_t i = 0; i < suite.groups.size(); ++i) { + const TestGroupDefinition& group = suite.groups[i]; + AppendLine(&output, 2, "- name: " + QuoteYaml(group.name)); + if (!group.description.empty()) { + AppendScalar(&output, 4, "description", group.description, + /*quote=*/true); + } + if (!group.depends_on.empty()) { + AppendScalar(&output, 4, "depends_on", + JoinQuotedList(group.depends_on), /*quote=*/false); + } + + AppendLine(&output, 4, "tests:"); + for (const TestCaseDefinition& test : group.tests) { + AppendLine(&output, 6, "- path: " + QuoteYaml(test.script_path)); + if (!test.name.empty() && test.name != test.script_path) { + AppendScalar(&output, 8, "name", test.name, /*quote=*/true); + } + if (!test.description.empty()) { + AppendScalar(&output, 8, "description", test.description, + /*quote=*/true); + } + if (!test.tags.empty()) { + AppendScalar(&output, 8, "tags", JoinQuotedList(test.tags), + /*quote=*/false); + } + if (!test.parameters.empty()) { + AppendLine(&output, 8, "parameters:"); + for (const auto& [key, value] : test.parameters) { + AppendScalar(&output, 10, key, value, /*quote=*/true); + } + } + } + + if (!group.tests.empty() && i + 1 < suite.groups.size()) { + output.append("\n"); + } + } + + return output; +} + +absl::Status WriteTestSuiteToFile(const TestSuiteDefinition& suite, + const std::string& path, bool overwrite) { + std::filesystem::path output_path(path); + std::error_code ec; + if (!overwrite && std::filesystem::exists(output_path, ec)) { + if (!ec) { + return absl::AlreadyExistsError( + absl::StrCat("Test suite file already exists: ", path)); + } + } + + std::filesystem::path parent = output_path.parent_path(); + if (!parent.empty()) { + std::filesystem::create_directories(parent, ec); + if (ec) { + return absl::InternalError(absl::StrCat( + "Failed to create directories for ", path, ": ", ec.message())); + } + } + + std::ofstream stream(output_path, std::ios::out | std::ios::trunc); + if (!stream.is_open()) { + return absl::InternalError( + absl::StrCat("Failed to open file for writing: ", path)); + } + + std::string yaml = BuildTestSuiteYaml(suite); + stream << yaml; + stream.close(); + + if (!stream) { + return absl::InternalError( + absl::StrCat("Failed to write test suite to ", path)); + } + return absl::OkStatus(); +} + +} // namespace cli +} // namespace yaze diff --git a/src/cli/service/test_suite_writer.h b/src/cli/service/test_suite_writer.h new file mode 100644 index 00000000..a892da59 --- /dev/null +++ b/src/cli/service/test_suite_writer.h @@ -0,0 +1,26 @@ +#ifndef YAZE_CLI_SERVICE_TEST_SUITE_WRITER_H_ +#define YAZE_CLI_SERVICE_TEST_SUITE_WRITER_H_ + +#include + +#include "absl/status/status.h" +#include "cli/service/test_suite.h" + +namespace yaze { +namespace cli { + +// Serializes a TestSuiteDefinition into a YAML document that is accepted by +// ParseTestSuiteDefinition(). +std::string BuildTestSuiteYaml(const TestSuiteDefinition& suite); + +// Writes the suite definition to the supplied path, creating parent +// directories if necessary. When overwrite is false and the file already +// exists, an ALREADY_EXISTS error is returned. +absl::Status WriteTestSuiteToFile(const TestSuiteDefinition& suite, + const std::string& path, + bool overwrite = false); + +} // namespace cli +} // namespace yaze + +#endif // YAZE_CLI_SERVICE_TEST_SUITE_WRITER_H_ diff --git a/src/cli/z3ed.cmake b/src/cli/z3ed.cmake index 2efd7b19..11600b4f 100644 --- a/src/cli/z3ed.cmake +++ b/src/cli/z3ed.cmake @@ -56,6 +56,8 @@ add_executable( cli/service/test_suite_loader.h cli/service/test_suite_reporter.cc cli/service/test_suite_reporter.h + cli/service/test_suite_writer.cc + cli/service/test_suite_writer.h cli/service/gemini_ai_service.cc app/rom.cc app/core/project.cc