Refactor CLI Service Structure and Enhance AI Integration

- Restructured CLI service source files to improve organization, moving files into dedicated directories for better maintainability.
- Introduced new AI service components, including `AIService`, `MockAIService`, and `GeminiAIService`, to facilitate natural language command generation.
- Implemented `PolicyEvaluator` and `ProposalRegistry` for enhanced proposal management and policy enforcement in AI workflows.
- Updated CMake configurations to reflect new file paths and ensure proper linking of the restructured components.
- Enhanced test suite with new test workflow generation capabilities, improving the robustness of automated testing.

This commit significantly advances the architecture of the z3ed system, laying the groundwork for more sophisticated AI-driven features and streamlined development processes.
This commit is contained in:
scawful
2025-10-03 09:54:27 -04:00
parent b89dcca93f
commit 90ddc3d50c
45 changed files with 224 additions and 167 deletions

View File

@@ -0,0 +1,93 @@
#ifndef YAZE_CLI_SERVICE_TEST_SUITE_H_
#define YAZE_CLI_SERVICE_TEST_SUITE_H_
#include <map>
#include <string>
#include <vector>
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "cli/service/gui/gui_automation_client.h"
namespace yaze {
namespace cli {
struct TestSuiteConfig {
int timeout_seconds = 0;
int retry_on_failure = 0;
bool parallel_execution = false;
};
struct TestCaseDefinition {
std::string id;
std::string name;
std::string script_path;
std::string description;
std::string group_name;
std::vector<std::string> tags;
std::map<std::string, std::string> parameters;
};
struct TestGroupDefinition {
std::string name;
std::string description;
std::vector<std::string> depends_on;
std::vector<TestCaseDefinition> tests;
};
struct TestSuiteDefinition {
std::string name;
std::string description;
std::string version;
TestSuiteConfig config;
std::vector<TestGroupDefinition> groups;
const TestGroupDefinition* FindGroup(absl::string_view group_name) const;
};
inline const TestGroupDefinition* TestSuiteDefinition::FindGroup(
absl::string_view group_name) const {
for (const auto& group : groups) {
if (group.name == group_name) {
return &group;
}
}
return nullptr;
}
enum class TestCaseOutcome {
kPassed,
kFailed,
kError,
kSkipped
};
struct TestCaseRunResult {
const TestCaseDefinition* test = nullptr;
const TestGroupDefinition* group = nullptr;
TestCaseOutcome outcome = TestCaseOutcome::kError;
absl::Time start_time = absl::InfinitePast();
absl::Duration duration = absl::ZeroDuration();
int attempts = 0;
int retries = 0;
std::string message;
std::string replay_session_id;
std::vector<AssertionOutcome> assertions;
std::vector<std::string> logs;
};
struct TestSuiteRunSummary {
const TestSuiteDefinition* suite = nullptr;
std::vector<TestCaseRunResult> results;
absl::Time started_at = absl::Now();
absl::Duration total_duration = absl::ZeroDuration();
int passed = 0;
int failed = 0;
int errors = 0;
int skipped = 0;
};
} // namespace cli
} // namespace yaze
#endif // YAZE_CLI_SERVICE_TEST_SUITE_H_

View File

@@ -0,0 +1,615 @@
#include "cli/service/testing/test_suite_loader.h"
#include <filesystem>
#include <fstream>
#include <sstream>
#include "absl/status/status.h"
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/numbers.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_split.h"
#include "absl/strings/strip.h"
#include "absl/strings/string_view.h"
#include "util/macro.h"
namespace yaze {
namespace cli {
namespace {
using ::absl::string_view;
std::string Trim(string_view value) {
return std::string(absl::StripAsciiWhitespace(value));
}
std::string StripComment(string_view line) {
bool in_single_quote = false;
bool in_double_quote = false;
for (size_t i = 0; i < line.size(); ++i) {
char c = line[i];
if (c == '\'') {
if (!in_double_quote) {
in_single_quote = !in_single_quote;
}
} else if (c == '"') {
if (!in_single_quote) {
in_double_quote = !in_double_quote;
}
} else if (c == '#' && !in_single_quote && !in_double_quote) {
return std::string(line.substr(0, i));
}
}
return std::string(line);
}
int CountIndent(string_view line) {
int count = 0;
for (char c : line) {
if (c == ' ') {
++count;
} else if (c == '\t') {
count += 2; // Treat tab as two spaces for simplicity
} else {
break;
}
}
return count;
}
bool ParseKeyValue(string_view input, std::string* key, std::string* value) {
size_t colon_pos = input.find(':');
if (colon_pos == string_view::npos) {
return false;
}
*key = Trim(input.substr(0, colon_pos));
*value = Trim(input.substr(colon_pos + 1));
return true;
}
std::string Unquote(string_view value) {
std::string trimmed = Trim(value);
if (trimmed.size() >= 2) {
if ((trimmed.front() == '"' && trimmed.back() == '"') ||
(trimmed.front() == '\'' && trimmed.back() == '\'')) {
return std::string(trimmed.substr(1, trimmed.size() - 2));
}
}
return trimmed;
}
std::vector<std::string> ParseInlineList(string_view value) {
std::vector<std::string> items;
std::string trimmed = Trim(value);
if (trimmed.empty()) {
return items;
}
if (trimmed.front() != '[' || trimmed.back() != ']') {
items.push_back(Unquote(trimmed));
return items;
}
string_view inner(trimmed.data() + 1, trimmed.size() - 2);
if (inner.empty()) {
return items;
}
for (string_view piece : absl::StrSplit(inner, ',', absl::SkipWhitespace())) {
items.push_back(Unquote(piece));
}
return items;
}
absl::StatusOr<int> ParseInt(string_view value) {
int result = 0;
if (!absl::SimpleAtoi(value, &result)) {
return absl::InvalidArgumentError(
absl::StrCat("Expected integer value, got '", value, "'"));
}
return result;
}
absl::StatusOr<int> ParseDurationSeconds(string_view value) {
std::string lower = absl::AsciiStrToLower(std::string(value));
if (lower.empty()) {
return 0;
}
int multiplier = 1;
if (absl::EndsWith(lower, "ms")) {
lower = lower.substr(0, lower.size() - 2);
multiplier = 0; // Use 0 to indicate sub-second resolution
} else if (absl::EndsWith(lower, "s")) {
lower = lower.substr(0, lower.size() - 1);
} else if (absl::EndsWith(lower, "m")) {
lower = lower.substr(0, lower.size() - 1);
multiplier = 60;
}
int numeric = 0;
if (!absl::SimpleAtoi(lower, &numeric)) {
return absl::InvalidArgumentError(
absl::StrCat("Invalid duration value '", value, "'"));
}
if (multiplier == 0) {
// Round milliseconds up to nearest whole second
return (numeric + 999) / 1000;
}
return numeric * multiplier;
}
bool ParseBoolean(string_view value, bool* output) {
std::string lower = absl::AsciiStrToLower(std::string(value));
if (lower == "true" || lower == "yes" || lower == "on" || lower == "1") {
*output = true;
return true;
}
if (lower == "false" || lower == "no" || lower == "off" || lower == "0") {
*output = false;
return true;
}
return false;
}
std::string DeriveTestName(const std::string& path) {
std::filesystem::path fs_path(path);
std::string stem = fs_path.stem().string();
if (stem.empty()) {
return path;
}
return stem;
}
absl::Status ParseScalarConfig(const std::string& key, const std::string& value,
TestSuiteConfig* config) {
if (key == "timeout_per_test") {
ASSIGN_OR_RETURN(int seconds, ParseDurationSeconds(value));
config->timeout_seconds = seconds;
return absl::OkStatus();
}
if (key == "retry_on_failure") {
ASSIGN_OR_RETURN(int retries, ParseInt(value));
config->retry_on_failure = retries;
return absl::OkStatus();
}
if (key == "parallel_execution") {
bool enabled = false;
if (!ParseBoolean(value, &enabled)) {
return absl::InvalidArgumentError(
absl::StrCat("Invalid boolean for parallel_execution: '", value,
"'"));
}
config->parallel_execution = enabled;
return absl::OkStatus();
}
return absl::InvalidArgumentError(
absl::StrCat("Unknown config key: '", key, "'"));
}
absl::Status ParseStringListBlock(const std::vector<std::string>& lines,
size_t* index, int base_indent,
std::vector<std::string>* output) {
while (*index < lines.size()) {
std::string raw = StripComment(lines[*index]);
std::string trimmed = Trim(raw);
int indent = CountIndent(raw);
if (trimmed.empty()) {
++(*index);
continue;
}
if (indent < base_indent) {
break;
}
if (indent != base_indent) {
return absl::InvalidArgumentError(
"Invalid indentation in list block");
}
if (trimmed.empty() || trimmed.front() != '-') {
return absl::InvalidArgumentError("Expected list entry starting with '-'");
}
std::string value = Trim(trimmed.substr(1));
output->push_back(Unquote(value));
++(*index);
}
return absl::OkStatus();
}
absl::Status ParseParametersBlock(const std::vector<std::string>& lines,
size_t* index, int base_indent,
std::map<std::string, std::string>* params) {
while (*index < lines.size()) {
std::string raw = StripComment(lines[*index]);
std::string trimmed = Trim(raw);
int indent = CountIndent(raw);
if (trimmed.empty()) {
++(*index);
continue;
}
if (indent < base_indent) {
break;
}
if (indent != base_indent) {
return absl::InvalidArgumentError("Invalid indentation in parameters block");
}
std::string key;
std::string value;
if (!ParseKeyValue(trimmed, &key, &value)) {
return absl::InvalidArgumentError(
"Expected key/value pair inside parameters block");
}
(*params)[key] = Unquote(value);
++(*index);
}
return absl::OkStatus();
}
absl::Status ParseTestCaseEntry(const std::vector<std::string>& lines,
size_t* index, int base_indent,
TestGroupDefinition* group) {
const std::string& raw_line = lines[*index];
std::string stripped = StripComment(raw_line);
int indent = CountIndent(stripped);
if (indent != base_indent) {
return absl::InvalidArgumentError("Invalid indentation for test case entry");
}
size_t dash_pos = stripped.find('-');
if (dash_pos == std::string::npos) {
return absl::InvalidArgumentError("Malformed list entry in tests block");
}
std::string content = Trim(stripped.substr(dash_pos + 1));
TestCaseDefinition test;
test.group_name = group->name;
auto commit_test = [&]() {
if (test.script_path.empty()) {
return absl::InvalidArgumentError("Test case missing script_path");
}
if (test.name.empty()) {
test.name = DeriveTestName(test.script_path);
}
if (test.id.empty()) {
test.id = absl::StrCat(test.group_name, ":", test.name);
}
group->tests.push_back(std::move(test));
return absl::OkStatus();
};
if (content.empty()) {
++(*index);
} else if (content.find(':') == std::string::npos) {
test.script_path = Unquote(content);
++(*index);
return commit_test();
} else {
std::string key;
std::string value;
if (!ParseKeyValue(content, &key, &value)) {
return absl::InvalidArgumentError("Malformed key/value in test entry");
}
if (key == "path" || key == "script" || key == "script_path") {
test.script_path = Unquote(value);
} else if (key == "name") {
test.name = Unquote(value);
} else if (key == "description") {
test.description = Unquote(value);
} else if (key == "id") {
test.id = Unquote(value);
} else if (key == "tags") {
auto tags = ParseInlineList(value);
test.tags.insert(test.tags.end(), tags.begin(), tags.end());
} else {
test.parameters[key] = Unquote(value);
}
++(*index);
}
while (*index < lines.size()) {
std::string raw = StripComment(lines[*index]);
std::string trimmed = Trim(raw);
int indent_next = CountIndent(raw);
if (trimmed.empty()) {
++(*index);
continue;
}
if (indent_next <= base_indent) {
break;
}
if (indent_next == base_indent + 2) {
std::string key;
std::string value;
if (!ParseKeyValue(trimmed, &key, &value)) {
return absl::InvalidArgumentError(
"Expected key/value pair in test definition");
}
if (key == "path" || key == "script" || key == "script_path") {
test.script_path = Unquote(value);
} else if (key == "name") {
test.name = Unquote(value);
} else if (key == "description") {
test.description = Unquote(value);
} else if (key == "id") {
test.id = Unquote(value);
} else if (key == "tags") {
auto tags = ParseInlineList(value);
if (tags.empty() && value.empty()) {
++(*index);
RETURN_IF_ERROR(ParseStringListBlock(lines, index, indent_next + 2,
&test.tags));
continue;
}
test.tags.insert(test.tags.end(), tags.begin(), tags.end());
} else if (key == "parameters") {
if (!value.empty()) {
return absl::InvalidArgumentError(
"parameters block must be indented on following lines");
}
++(*index);
RETURN_IF_ERROR(ParseParametersBlock(lines, index, indent_next + 2,
&test.parameters));
continue;
} else {
test.parameters[key] = Unquote(value);
}
++(*index);
} else {
return absl::InvalidArgumentError(
"Unexpected indentation inside test entry");
}
}
return commit_test();
}
absl::Status ParseTestsBlock(const std::vector<std::string>& lines,
size_t* index, int base_indent,
TestGroupDefinition* group) {
while (*index < lines.size()) {
std::string raw = StripComment(lines[*index]);
std::string trimmed = Trim(raw);
int indent = CountIndent(raw);
if (trimmed.empty()) {
++(*index);
continue;
}
if (indent < base_indent) {
break;
}
if (indent != base_indent) {
return absl::InvalidArgumentError(
"Invalid indentation inside tests block");
}
RETURN_IF_ERROR(ParseTestCaseEntry(lines, index, base_indent, group));
}
return absl::OkStatus();
}
absl::Status ParseGroupEntry(const std::vector<std::string>& lines,
size_t* index, TestSuiteDefinition* suite) {
const std::string& raw_line = lines[*index];
std::string stripped = StripComment(raw_line);
int base_indent = CountIndent(stripped);
size_t dash_pos = stripped.find('-');
if (dash_pos == std::string::npos || dash_pos < base_indent) {
return absl::InvalidArgumentError("Expected '-' to start group entry");
}
std::string content = Trim(stripped.substr(dash_pos + 1));
TestGroupDefinition group;
if (!content.empty()) {
std::string key;
std::string value;
if (!ParseKeyValue(content, &key, &value)) {
return absl::InvalidArgumentError("Malformed group entry");
}
if (key == "name") {
group.name = Unquote(value);
} else if (key == "description") {
group.description = Unquote(value);
} else if (key == "depends_on") {
auto deps = ParseInlineList(value);
group.depends_on.insert(group.depends_on.end(), deps.begin(), deps.end());
} else {
return absl::InvalidArgumentError(
absl::StrCat("Unknown field in group entry: '", key, "'"));
}
}
++(*index);
while (*index < lines.size()) {
std::string raw = StripComment(lines[*index]);
std::string trimmed = Trim(raw);
int indent = CountIndent(raw);
if (trimmed.empty()) {
++(*index);
continue;
}
if (indent <= base_indent) {
break;
}
if (indent == base_indent + 2) {
std::string key;
std::string value;
if (!ParseKeyValue(trimmed, &key, &value)) {
return absl::InvalidArgumentError(
"Expected key/value pair inside group definition");
}
if (key == "name") {
group.name = Unquote(value);
++(*index);
} else if (key == "description") {
group.description = Unquote(value);
++(*index);
} else if (key == "depends_on") {
if (!value.empty()) {
auto deps = ParseInlineList(value);
group.depends_on.insert(group.depends_on.end(), deps.begin(),
deps.end());
++(*index);
} else {
++(*index);
RETURN_IF_ERROR(ParseStringListBlock(lines, index, base_indent + 4,
&group.depends_on));
}
} else if (key == "tests") {
if (!value.empty()) {
return absl::InvalidArgumentError(
"tests block must be defined as indented list");
}
++(*index);
RETURN_IF_ERROR(
ParseTestsBlock(lines, index, base_indent + 4, &group));
} else {
return absl::InvalidArgumentError(
absl::StrCat("Unknown attribute in group definition: '", key,
"'"));
}
} else {
return absl::InvalidArgumentError(
"Unexpected indentation inside group definition");
}
}
if (group.name.empty()) {
return absl::InvalidArgumentError(
"Each test group must define a name");
}
suite->groups.push_back(std::move(group));
return absl::OkStatus();
}
absl::Status ParseGroupBlock(const std::vector<std::string>& lines,
size_t* index, TestSuiteDefinition* suite) {
while (*index < lines.size()) {
std::string raw = StripComment(lines[*index]);
std::string trimmed = Trim(raw);
int indent = CountIndent(raw);
if (trimmed.empty()) {
++(*index);
continue;
}
if (indent < 2) {
break;
}
if (indent != 2) {
return absl::InvalidArgumentError(
"Invalid indentation inside test_groups block");
}
RETURN_IF_ERROR(ParseGroupEntry(lines, index, suite));
}
return absl::OkStatus();
}
absl::Status ParseConfigBlock(const std::vector<std::string>& lines,
size_t* index, TestSuiteConfig* config) {
while (*index < lines.size()) {
std::string raw = StripComment(lines[*index]);
std::string trimmed = Trim(raw);
int indent = CountIndent(raw);
if (trimmed.empty()) {
++(*index);
continue;
}
if (indent < 2) {
break;
}
if (indent != 2) {
return absl::InvalidArgumentError(
"Invalid indentation inside config block");
}
std::string key;
std::string value;
if (!ParseKeyValue(trimmed, &key, &value)) {
return absl::InvalidArgumentError(
"Expected key/value pair inside config block");
}
RETURN_IF_ERROR(ParseScalarConfig(key, value, config));
++(*index);
}
return absl::OkStatus();
}
} // namespace
absl::StatusOr<TestSuiteDefinition> ParseTestSuiteDefinition(
absl::string_view content) {
std::vector<std::string> lines = absl::StrSplit(content, '\n');
TestSuiteDefinition suite;
size_t index = 0;
while (index < lines.size()) {
std::string raw = StripComment(lines[index]);
std::string trimmed = Trim(raw);
if (trimmed.empty()) {
++index;
continue;
}
int indent = CountIndent(raw);
if (indent != 0) {
return absl::InvalidArgumentError(
"Top-level entries must not be indented in suite definition");
}
std::string key;
std::string value;
if (!ParseKeyValue(trimmed, &key, &value)) {
return absl::InvalidArgumentError(
absl::StrCat("Malformed top-level entry: '", trimmed, "'"));
}
if (key == "name") {
suite.name = Unquote(value);
++index;
} else if (key == "description") {
suite.description = Unquote(value);
++index;
} else if (key == "version") {
suite.version = Unquote(value);
++index;
} else if (key == "config") {
if (!value.empty()) {
return absl::InvalidArgumentError(
"config block must not specify inline value");
}
++index;
RETURN_IF_ERROR(ParseConfigBlock(lines, &index, &suite.config));
} else if (key == "test_groups") {
if (!value.empty()) {
return absl::InvalidArgumentError(
"test_groups must be defined as an indented list");
}
++index;
RETURN_IF_ERROR(ParseGroupBlock(lines, &index, &suite));
} else {
return absl::InvalidArgumentError(
absl::StrCat("Unknown top-level key: '", key, "'"));
}
}
if (suite.name.empty()) {
suite.name = "Unnamed Suite";
}
if (suite.version.empty()) {
suite.version = "1.0";
}
return suite;
}
absl::StatusOr<TestSuiteDefinition> LoadTestSuiteFromFile(
const std::string& path) {
std::ifstream file(path);
if (!file.is_open()) {
return absl::NotFoundError(
absl::StrCat("Failed to open test suite file '", path, "'"));
}
std::stringstream buffer;
buffer << file.rdbuf();
return ParseTestSuiteDefinition(buffer.str());
}
} // namespace cli
} // namespace yaze

View File

@@ -0,0 +1,19 @@
#ifndef YAZE_CLI_SERVICE_TEST_SUITE_LOADER_H_
#define YAZE_CLI_SERVICE_TEST_SUITE_LOADER_H_
#include <string>
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "cli/service/testing/test_suite.h"
namespace yaze {
namespace cli {
absl::StatusOr<TestSuiteDefinition> ParseTestSuiteDefinition(absl::string_view content);
absl::StatusOr<TestSuiteDefinition> LoadTestSuiteFromFile(const std::string& path);
} // namespace cli
} // namespace yaze
#endif // YAZE_CLI_SERVICE_TEST_SUITE_LOADER_H_

View File

@@ -0,0 +1,293 @@
#include "cli/service/testing/test_suite_reporter.h"
#include <filesystem>
#include <fstream>
#include <sstream>
#include "absl/status/status.h"
#include "absl/strings/ascii.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "util/macro.h"
namespace yaze {
namespace cli {
namespace {
std::string OutcomeToString(TestCaseOutcome outcome) {
switch (outcome) {
case TestCaseOutcome::kPassed:
return "PASS";
case TestCaseOutcome::kFailed:
return "FAIL";
case TestCaseOutcome::kError:
return "ERROR";
case TestCaseOutcome::kSkipped:
return "SKIP";
}
return "UNKNOWN";
}
std::string EscapeXml(absl::string_view input) {
std::string escaped;
escaped.reserve(input.size());
for (char c : input) {
switch (c) {
case '&':
escaped.append("&amp;");
break;
case '<':
escaped.append("&lt;");
break;
case '>':
escaped.append("&gt;");
break;
case '\"':
escaped.append("&quot;");
break;
case '\'':
escaped.append("&apos;");
break;
default:
escaped.push_back(c);
break;
}
}
return escaped;
}
std::string JoinLogs(const std::vector<std::string>& logs) {
if (logs.empty()) {
return {};
}
return absl::StrJoin(logs, "\n");
}
void ComputeSummaryCounters(const TestSuiteRunSummary& summary, int* total,
int* passed, int* failed, int* errors,
int* skipped) {
*total = static_cast<int>(summary.results.size());
*passed = summary.passed;
*failed = summary.failed;
*errors = summary.errors;
*skipped = summary.skipped;
if (*passed + *failed + *errors + *skipped != *total) {
*passed = *failed = *errors = *skipped = 0;
for (const auto& result : summary.results) {
switch (result.outcome) {
case TestCaseOutcome::kPassed:
++(*passed);
break;
case TestCaseOutcome::kFailed:
++(*failed);
break;
case TestCaseOutcome::kError:
++(*errors);
break;
case TestCaseOutcome::kSkipped:
++(*skipped);
break;
}
}
}
}
absl::Duration TotalDuration(const TestSuiteRunSummary& summary) {
if (summary.total_duration > absl::ZeroDuration()) {
return summary.total_duration;
}
absl::Duration total = absl::ZeroDuration();
for (const auto& result : summary.results) {
total += result.duration;
}
return total;
}
} // namespace
std::string BuildTextSummary(const TestSuiteRunSummary& summary) {
std::ostringstream oss;
int total = 0;
int passed = 0;
int failed = 0;
int errors = 0;
int skipped = 0;
ComputeSummaryCounters(summary, &total, &passed, &failed, &errors, &skipped);
absl::Time timestamp = summary.started_at;
if (timestamp == absl::InfinitePast()) {
timestamp = absl::Now();
}
oss << "Suite: "
<< (summary.suite && !summary.suite->name.empty() ? summary.suite->name
: "Unnamed Suite")
<< "\n";
oss << "Started: "
<< absl::FormatTime("%Y-%m-%d %H:%M:%S", timestamp, absl::UTCTimeZone())
<< " UTC\n";
oss << "Totals: " << total << " (" << passed << " passed, " << failed
<< " failed, " << errors << " errors, " << skipped << " skipped)";
absl::Duration duration = TotalDuration(summary);
if (duration > absl::ZeroDuration()) {
oss << " in "
<< absl::StrFormat("%.2fs", absl::ToDoubleSeconds(duration));
}
oss << "\n\n";
for (const auto& result : summary.results) {
std::string group_name;
if (result.group) {
group_name = result.group->name;
} else if (result.test) {
group_name = result.test->group_name;
}
std::string test_name = result.test ? result.test->name : "<unnamed>";
oss << " [" << OutcomeToString(result.outcome) << "] ";
if (!group_name.empty()) {
oss << group_name << " :: ";
}
oss << test_name;
if (result.duration > absl::ZeroDuration()) {
oss << " (" << absl::StrFormat("%.2fs",
absl::ToDoubleSeconds(result.duration))
<< ")";
}
oss << "\n";
if (!result.message.empty() &&
result.outcome != TestCaseOutcome::kPassed) {
oss << " " << result.message << "\n";
}
if (!result.assertions.empty() &&
result.outcome != TestCaseOutcome::kPassed) {
for (const auto& assertion : result.assertions) {
oss << " - " << assertion.description << " : "
<< (assertion.passed ? "PASS" : "FAIL");
if (!assertion.error_message.empty()) {
oss << " (" << assertion.error_message << ")";
}
oss << "\n";
}
}
}
return oss.str();
}
absl::StatusOr<std::string> BuildJUnitReport(
const TestSuiteRunSummary& summary) {
std::ostringstream oss;
int total = 0;
int passed = 0;
int failed = 0;
int errors = 0;
int skipped = 0;
ComputeSummaryCounters(summary, &total, &passed, &failed, &errors, &skipped);
absl::Time timestamp = summary.started_at;
if (timestamp == absl::InfinitePast()) {
timestamp = absl::Now();
}
absl::Duration duration = TotalDuration(summary);
std::string suite_name =
summary.suite ? summary.suite->name : "YAZE GUI Test Suite";
oss << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
oss << "<testsuite name=\"" << EscapeXml(suite_name) << "\" tests=\""
<< total << "\" failures=\"" << failed << "\" errors=\"" << errors
<< "\" skipped=\"" << skipped << "\" time=\""
<< absl::StrFormat("%.3f", absl::ToDoubleSeconds(duration))
<< "\" timestamp=\""
<< EscapeXml(
absl::FormatTime("%Y-%m-%dT%H:%M:%SZ", timestamp,
absl::UTCTimeZone()))
<< "\">\n";
for (const auto& result : summary.results) {
std::string classname;
if (result.group) {
classname = result.group->name;
} else if (result.test) {
classname = result.test->group_name;
}
std::string test_name = result.test ? result.test->name : "Test";
oss << " <testcase classname=\"" << EscapeXml(classname)
<< "\" name=\"" << EscapeXml(test_name) << "\" time=\""
<< absl::StrFormat("%.3f", absl::ToDoubleSeconds(result.duration))
<< "\">";
if (result.outcome == TestCaseOutcome::kFailed) {
std::string body = result.message;
if (!result.assertions.empty()) {
std::vector<std::string> assertion_lines;
for (const auto& assertion : result.assertions) {
assertion_lines.push_back(
absl::StrCat(assertion.description, " => ",
assertion.passed ? "PASS" : "FAIL"));
}
body = absl::StrCat(body, "\n",
absl::StrJoin(assertion_lines, "\n"));
}
oss << "\n <failure message=\"" << EscapeXml(result.message)
<< "\">" << EscapeXml(body) << "</failure>";
if (!result.logs.empty()) {
oss << "\n <system-out>" << EscapeXml(JoinLogs(result.logs))
<< "</system-out>";
}
oss << "\n </testcase>\n";
continue;
}
if (result.outcome == TestCaseOutcome::kError) {
std::string detail = result.message;
if (!result.logs.empty()) {
detail = absl::StrCat(detail, "\n", JoinLogs(result.logs));
}
oss << "\n <error message=\"" << EscapeXml(result.message)
<< "\">" << EscapeXml(detail) << "</error>";
oss << "\n </testcase>\n";
continue;
}
if (!result.logs.empty()) {
oss << "\n <system-out>" << EscapeXml(JoinLogs(result.logs))
<< "</system-out>";
}
oss << "\n </testcase>\n";
}
oss << "</testsuite>\n";
return oss.str();
}
absl::Status WriteJUnitReport(const TestSuiteRunSummary& summary,
const std::string& output_path) {
ASSIGN_OR_RETURN(std::string xml, BuildJUnitReport(summary));
std::filesystem::path path(output_path);
if (path.has_parent_path()) {
std::error_code ec;
std::filesystem::create_directories(path.parent_path(), ec);
if (ec) {
return absl::InternalError(
absl::StrCat("Failed to create directory for JUnit report: ",
ec.message()));
}
}
std::ofstream out(path);
if (!out.is_open()) {
return absl::InternalError(
absl::StrCat("Unable to open JUnit output file '", output_path,
"'"));
}
out << xml;
return absl::OkStatus();
}
} // namespace cli
} // namespace yaze

View File

@@ -0,0 +1,20 @@
#ifndef YAZE_CLI_SERVICE_TEST_SUITE_REPORTER_H_
#define YAZE_CLI_SERVICE_TEST_SUITE_REPORTER_H_
#include <string>
#include "absl/status/statusor.h"
#include "cli/service/testing/test_suite.h"
namespace yaze {
namespace cli {
std::string BuildTextSummary(const TestSuiteRunSummary& summary);
absl::StatusOr<std::string> BuildJUnitReport(const TestSuiteRunSummary& summary);
absl::Status WriteJUnitReport(const TestSuiteRunSummary& summary,
const std::string& output_path);
} // namespace cli
} // namespace yaze
#endif // YAZE_CLI_SERVICE_TEST_SUITE_REPORTER_H_

View File

@@ -0,0 +1,181 @@
#include "cli/service/testing/test_suite_writer.h"
#include <filesystem>
#include <fstream>
#include <system_error>
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
namespace yaze {
namespace cli {
namespace {
std::string Indent(int count) { return std::string(count, ' '); }
std::string QuoteYaml(absl::string_view value) {
std::string escaped(value);
absl::StrReplaceAll({{"\\", "\\\\"}, {"\"", "\\\""}}, &escaped);
return absl::StrCat("\"", escaped, "\"");
}
void AppendLine(std::string* out, int indent, absl::string_view line) {
out->append(Indent(indent));
out->append(line.data(), line.size());
out->append("\n");
}
void AppendScalar(std::string* out, int indent, absl::string_view key,
absl::string_view value, bool quote) {
out->append(Indent(indent));
out->append(key.data(), key.size());
out->append(":");
if (!value.empty()) {
out->append(" ");
if (quote) {
out->append(QuoteYaml(value));
} else {
out->append(value.data(), value.size());
}
}
out->append("\n");
}
std::string FormatDuration(int seconds) {
if (seconds <= 0) {
return "0s";
}
if (seconds % 60 == 0) {
return absl::StrCat(seconds / 60, "m");
}
return absl::StrCat(seconds, "s");
}
std::string FormatBool(bool value) { return value ? "true" : "false"; }
std::string JoinQuotedList(const std::vector<std::string>& values) {
if (values.empty()) {
return "[]";
}
std::vector<std::string> quoted;
quoted.reserve(values.size());
for (const auto& v : values) {
quoted.push_back(QuoteYaml(v));
}
return absl::StrCat("[", absl::StrJoin(quoted, ", "), "]");
}
} // namespace
std::string BuildTestSuiteYaml(const TestSuiteDefinition& suite) {
std::string output;
if (!suite.name.empty()) {
AppendScalar(&output, 0, "name", suite.name, /*quote=*/true);
} else {
AppendScalar(&output, 0, "name", "Unnamed Suite", /*quote=*/true);
}
if (!suite.description.empty()) {
AppendScalar(&output, 0, "description", suite.description,
/*quote=*/true);
}
if (!suite.version.empty()) {
AppendScalar(&output, 0, "version", suite.version, /*quote=*/true);
}
AppendLine(&output, 0, "config:");
AppendScalar(&output, 2, "timeout_per_test",
FormatDuration(suite.config.timeout_seconds),
/*quote=*/false);
AppendScalar(&output, 2, "retry_on_failure",
absl::StrCat(suite.config.retry_on_failure),
/*quote=*/false);
AppendScalar(&output, 2, "parallel_execution",
FormatBool(suite.config.parallel_execution),
/*quote=*/false);
AppendLine(&output, 0, "test_groups:");
for (size_t i = 0; i < suite.groups.size(); ++i) {
const TestGroupDefinition& group = suite.groups[i];
AppendLine(&output, 2, "- name: " + QuoteYaml(group.name));
if (!group.description.empty()) {
AppendScalar(&output, 4, "description", group.description,
/*quote=*/true);
}
if (!group.depends_on.empty()) {
AppendScalar(&output, 4, "depends_on",
JoinQuotedList(group.depends_on), /*quote=*/false);
}
AppendLine(&output, 4, "tests:");
for (const TestCaseDefinition& test : group.tests) {
AppendLine(&output, 6, "- path: " + QuoteYaml(test.script_path));
if (!test.name.empty() && test.name != test.script_path) {
AppendScalar(&output, 8, "name", test.name, /*quote=*/true);
}
if (!test.description.empty()) {
AppendScalar(&output, 8, "description", test.description,
/*quote=*/true);
}
if (!test.tags.empty()) {
AppendScalar(&output, 8, "tags", JoinQuotedList(test.tags),
/*quote=*/false);
}
if (!test.parameters.empty()) {
AppendLine(&output, 8, "parameters:");
for (const auto& [key, value] : test.parameters) {
AppendScalar(&output, 10, key, value, /*quote=*/true);
}
}
}
if (!group.tests.empty() && i + 1 < suite.groups.size()) {
output.append("\n");
}
}
return output;
}
absl::Status WriteTestSuiteToFile(const TestSuiteDefinition& suite,
const std::string& path, bool overwrite) {
std::filesystem::path output_path(path);
std::error_code ec;
if (!overwrite && std::filesystem::exists(output_path, ec)) {
if (!ec) {
return absl::AlreadyExistsError(
absl::StrCat("Test suite file already exists: ", path));
}
}
std::filesystem::path parent = output_path.parent_path();
if (!parent.empty()) {
std::filesystem::create_directories(parent, ec);
if (ec) {
return absl::InternalError(absl::StrCat(
"Failed to create directories for ", path, ": ", ec.message()));
}
}
std::ofstream stream(output_path, std::ios::out | std::ios::trunc);
if (!stream.is_open()) {
return absl::InternalError(
absl::StrCat("Failed to open file for writing: ", path));
}
std::string yaml = BuildTestSuiteYaml(suite);
stream << yaml;
stream.close();
if (!stream) {
return absl::InternalError(
absl::StrCat("Failed to write test suite to ", path));
}
return absl::OkStatus();
}
} // namespace cli
} // namespace yaze

View File

@@ -0,0 +1,26 @@
#ifndef YAZE_CLI_SERVICE_TEST_SUITE_WRITER_H_
#define YAZE_CLI_SERVICE_TEST_SUITE_WRITER_H_
#include <string>
#include "absl/status/status.h"
#include "cli/service/testing/test_suite.h"
namespace yaze {
namespace cli {
// Serializes a TestSuiteDefinition into a YAML document that is accepted by
// ParseTestSuiteDefinition().
std::string BuildTestSuiteYaml(const TestSuiteDefinition& suite);
// Writes the suite definition to the supplied path, creating parent
// directories if necessary. When overwrite is false and the file already
// exists, an ALREADY_EXISTS error is returned.
absl::Status WriteTestSuiteToFile(const TestSuiteDefinition& suite,
const std::string& path,
bool overwrite = false);
} // namespace cli
} // namespace yaze
#endif // YAZE_CLI_SERVICE_TEST_SUITE_WRITER_H_

View File

@@ -0,0 +1,227 @@
// test_workflow_generator.cc
// Implementation of natural language to test workflow conversion
#include "cli/service/testing/test_workflow_generator.h"
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_replace.h"
#include <regex>
namespace yaze {
namespace cli {
std::string TestStep::ToString() const {
switch (type) {
case TestStepType::kClick:
return absl::StrFormat("Click(%s)", target);
case TestStepType::kType:
return absl::StrFormat("Type(%s, \"%s\"%s)", target, text,
clear_first ? ", clear_first" : "");
case TestStepType::kWait:
return absl::StrFormat("Wait(%s, %dms)", condition, timeout_ms);
case TestStepType::kAssert:
return absl::StrFormat("Assert(%s)", condition);
case TestStepType::kScreenshot:
return "Screenshot()";
}
return "Unknown";
}
std::string TestWorkflow::ToString() const {
std::string result = absl::StrCat("Workflow: ", description, "\n");
for (size_t i = 0; i < steps.size(); ++i) {
absl::StrAppend(&result, " ", i + 1, ". ", steps[i].ToString(), "\n");
}
return result;
}
absl::StatusOr<TestWorkflow> TestWorkflowGenerator::GenerateWorkflow(
const std::string& prompt) {
std::string normalized_prompt = absl::AsciiStrToLower(prompt);
// Try pattern matching in order of specificity
std::string editor_name, input_name, text, button_name;
// Pattern 1: "Open <Editor> and verify it loads"
if (MatchesOpenAndVerify(normalized_prompt, &editor_name)) {
return BuildOpenAndVerifyWorkflow(editor_name);
}
// Pattern 2: "Open <Editor> editor"
if (MatchesOpenEditor(normalized_prompt, &editor_name)) {
return BuildOpenEditorWorkflow(editor_name);
}
// Pattern 3: "Type '<text>' in <input>"
if (MatchesTypeInput(normalized_prompt, &input_name, &text)) {
return BuildTypeInputWorkflow(input_name, text);
}
// Pattern 4: "Click <button>"
if (MatchesClickButton(normalized_prompt, &button_name)) {
return BuildClickButtonWorkflow(button_name);
}
// If no patterns match, return helpful error
return absl::InvalidArgumentError(
absl::StrFormat(
"Unable to parse prompt: \"%s\"\n\n"
"Supported patterns:\n"
" - Open <Editor> editor\n"
" - Open <Editor> and verify it loads\n"
" - Type '<text>' in <input>\n"
" - Click <button>\n\n"
"Examples:\n"
" - Open Overworld editor\n"
" - Open Dungeon editor and verify it loads\n"
" - Type 'zelda3.sfc' in filename input\n"
" - Click Open ROM button",
prompt));
}
bool TestWorkflowGenerator::MatchesOpenEditor(const std::string& prompt,
std::string* editor_name) {
// Match: "open <name> editor" or "open <name>"
std::regex pattern(R"(open\s+(\w+)(?:\s+editor)?)");
std::smatch match;
if (std::regex_search(prompt, match, pattern) && match.size() > 1) {
*editor_name = match[1].str();
return true;
}
return false;
}
bool TestWorkflowGenerator::MatchesOpenAndVerify(const std::string& prompt,
std::string* editor_name) {
// Match: "open <name> and verify" or "open <name> editor and verify it loads"
std::regex pattern(R"(open\s+(\w+)(?:\s+editor)?\s+and\s+verify)");
std::smatch match;
if (std::regex_search(prompt, match, pattern) && match.size() > 1) {
*editor_name = match[1].str();
return true;
}
return false;
}
bool TestWorkflowGenerator::MatchesTypeInput(const std::string& prompt,
std::string* input_name,
std::string* text) {
// Match: "type 'text' in <input>" or "type \"text\" in <input>"
std::regex pattern(R"(type\s+['"]([^'"]+)['"]\s+in(?:to)?\s+(\w+))");
std::smatch match;
if (std::regex_search(prompt, match, pattern) && match.size() > 2) {
*text = match[1].str();
*input_name = match[2].str();
return true;
}
return false;
}
bool TestWorkflowGenerator::MatchesClickButton(const std::string& prompt,
std::string* button_name) {
// Match: "click <button>" or "click <button> button"
std::regex pattern(R"(click\s+([\w\s]+?)(?:\s+button)?\s*$)");
std::smatch match;
if (std::regex_search(prompt, match, pattern) && match.size() > 1) {
*button_name = match[1].str();
return true;
}
return false;
}
std::string TestWorkflowGenerator::NormalizeEditorName(const std::string& name) {
std::string normalized = name;
// Capitalize first letter
if (!normalized.empty()) {
normalized[0] = std::toupper(normalized[0]);
}
// Add " Editor" suffix if not present
if (!absl::StrContains(absl::AsciiStrToLower(normalized), "editor")) {
absl::StrAppend(&normalized, " Editor");
}
return normalized;
}
TestWorkflow TestWorkflowGenerator::BuildOpenEditorWorkflow(
const std::string& editor_name) {
std::string normalized_name = NormalizeEditorName(editor_name);
TestWorkflow workflow;
workflow.description = absl::StrFormat("Open %s", normalized_name);
// Step 1: Click the editor button
TestStep click_step;
click_step.type = TestStepType::kClick;
click_step.target = absl::StrFormat("button:%s",
absl::StrReplaceAll(normalized_name,
{{" Editor", ""}}));
workflow.steps.push_back(click_step);
// Step 2: Wait for editor window to appear
TestStep wait_step;
wait_step.type = TestStepType::kWait;
wait_step.condition = absl::StrFormat("window_visible:%s", normalized_name);
wait_step.timeout_ms = 5000;
workflow.steps.push_back(wait_step);
return workflow;
}
TestWorkflow TestWorkflowGenerator::BuildOpenAndVerifyWorkflow(
const std::string& editor_name) {
// Start with basic open workflow
TestWorkflow workflow = BuildOpenEditorWorkflow(editor_name);
workflow.description = absl::StrFormat("Open and verify %s",
NormalizeEditorName(editor_name));
// Add assertion step
TestStep assert_step;
assert_step.type = TestStepType::kAssert;
assert_step.condition = absl::StrFormat("visible:%s",
NormalizeEditorName(editor_name));
workflow.steps.push_back(assert_step);
return workflow;
}
TestWorkflow TestWorkflowGenerator::BuildTypeInputWorkflow(
const std::string& input_name, const std::string& text) {
TestWorkflow workflow;
workflow.description = absl::StrFormat("Type '%s' into %s", text, input_name);
// Step 1: Click input to focus
TestStep click_step;
click_step.type = TestStepType::kClick;
click_step.target = absl::StrFormat("input:%s", input_name);
workflow.steps.push_back(click_step);
// Step 2: Type the text
TestStep type_step;
type_step.type = TestStepType::kType;
type_step.target = absl::StrFormat("input:%s", input_name);
type_step.text = text;
type_step.clear_first = true;
workflow.steps.push_back(type_step);
return workflow;
}
TestWorkflow TestWorkflowGenerator::BuildClickButtonWorkflow(
const std::string& button_name) {
TestWorkflow workflow;
workflow.description = absl::StrFormat("Click '%s' button", button_name);
TestStep click_step;
click_step.type = TestStepType::kClick;
click_step.target = absl::StrFormat("button:%s", button_name);
workflow.steps.push_back(click_step);
return workflow;
}
} // namespace cli
} // namespace yaze

View File

@@ -0,0 +1,106 @@
// test_workflow_generator.h
// Converts natural language prompts into GUI automation workflows
#ifndef YAZE_CLI_SERVICE_TEST_WORKFLOW_GENERATOR_H
#define YAZE_CLI_SERVICE_TEST_WORKFLOW_GENERATOR_H
#include "absl/status/statusor.h"
#include <string>
#include <vector>
namespace yaze {
namespace cli {
/**
* @brief Type of test step to execute
*/
enum class TestStepType {
kClick, // Click a button or element
kType, // Type text into an input
kWait, // Wait for a condition
kAssert, // Assert a condition is true
kScreenshot // Capture a screenshot
};
/**
* @brief A single step in a GUI test workflow
*/
struct TestStep {
TestStepType type;
std::string target; // Widget/element target (e.g., "button:Overworld")
std::string text; // Text to type (for kType steps)
std::string condition; // Condition to wait for or assert
int timeout_ms = 5000; // Timeout for wait operations
bool clear_first = false; // Clear text before typing
std::string ToString() const;
};
/**
* @brief A complete GUI test workflow
*/
struct TestWorkflow {
std::string description;
std::vector<TestStep> steps;
std::string ToString() const;
};
/**
* @brief Generates GUI test workflows from natural language prompts
*
* This class uses pattern matching to convert user prompts into
* structured test workflows that can be executed by GuiAutomationClient.
*
* Example prompts:
* - "Open Overworld editor" → Click button, Wait for window
* - "Open Dungeon editor and verify it loads" → Click, Wait, Assert
* - "Type 'zelda3.sfc' in filename input" → Click input, Type text
*
* Usage:
* @code
* TestWorkflowGenerator generator;
* auto workflow = generator.GenerateWorkflow("Open Overworld editor");
* if (!workflow.ok()) return workflow.status();
*
* for (const auto& step : workflow->steps) {
* std::cout << step.ToString() << "\n";
* }
* @endcode
*/
class TestWorkflowGenerator {
public:
TestWorkflowGenerator() = default;
/**
* @brief Generate a test workflow from a natural language prompt
* @param prompt Natural language description of desired GUI actions
* @return TestWorkflow or error if prompt is unsupported
*/
absl::StatusOr<TestWorkflow> GenerateWorkflow(const std::string& prompt);
private:
// Pattern matchers for different prompt types
bool MatchesOpenEditor(const std::string& prompt, std::string* editor_name);
bool MatchesOpenAndVerify(const std::string& prompt, std::string* editor_name);
bool MatchesTypeInput(const std::string& prompt, std::string* input_name,
std::string* text);
bool MatchesClickButton(const std::string& prompt, std::string* button_name);
bool MatchesMultiStep(const std::string& prompt);
// Workflow builders
TestWorkflow BuildOpenEditorWorkflow(const std::string& editor_name);
TestWorkflow BuildOpenAndVerifyWorkflow(const std::string& editor_name);
TestWorkflow BuildTypeInputWorkflow(const std::string& input_name,
const std::string& text);
TestWorkflow BuildClickButtonWorkflow(const std::string& button_name);
// Helper to normalize editor names (e.g., "overworld" → "Overworld Editor")
std::string NormalizeEditorName(const std::string& name);
};
} // namespace cli
} // namespace yaze
#endif // YAZE_CLI_SERVICE_TEST_WORKFLOW_GENERATOR_H