feat: Add GUI automation client and test workflow generator
- Implemented GuiAutomationClient for gRPC communication with the test harness. - Added methods for various GUI actions: Click, Type, Wait, Assert, and Screenshot. - Created TestWorkflowGenerator to convert natural language prompts into structured test workflows. - Enhanced HandleTestCommand to support new command-line arguments for GUI automation. - Updated CMakeLists.txt to include new source files for GUI automation and workflow generation.
This commit is contained in:
scawful
@@ -172,6 +172,8 @@ if (YAZE_BUILD_LIB)
|
||||
# CLI service sources (needed for ProposalDrawer)
|
||||
cli/service/proposal_registry.cc
|
||||
cli/service/rom_sandbox_manager.cc
|
||||
cli/service/gui_automation_client.cc
|
||||
cli/service/test_workflow_generator.cc
|
||||
)
|
||||
|
||||
# Create full library for C API
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include "cli/service/proposal_registry.h"
|
||||
#include "cli/service/resource_catalog.h"
|
||||
#include "cli/service/rom_sandbox_manager.h"
|
||||
#include "cli/service/gui_automation_client.h"
|
||||
#include "cli/service/test_workflow_generator.h"
|
||||
#include "util/macro.h"
|
||||
|
||||
#include "absl/flags/declare.h"
|
||||
@@ -352,88 +354,131 @@ absl::Status HandleDiffCommand(Rom& rom, const std::vector<std::string>& args) {
|
||||
}
|
||||
|
||||
absl::Status HandleTestCommand(const std::vector<std::string>& arg_vec) {
|
||||
if (arg_vec.size() < 2 || arg_vec[0] != "--test") {
|
||||
return absl::InvalidArgumentError("Usage: agent test --test <test_name>");
|
||||
// Parse arguments
|
||||
std::string prompt;
|
||||
std::string host = "localhost";
|
||||
int port = 50052;
|
||||
int timeout_sec = 30;
|
||||
|
||||
for (size_t i = 0; i < arg_vec.size(); ++i) {
|
||||
const std::string& token = arg_vec[i];
|
||||
|
||||
if (token == "--prompt" && i + 1 < arg_vec.size()) {
|
||||
prompt = arg_vec[++i];
|
||||
} else if (token == "--host" && i + 1 < arg_vec.size()) {
|
||||
host = arg_vec[++i];
|
||||
} else if (token == "--port" && i + 1 < arg_vec.size()) {
|
||||
port = std::stoi(arg_vec[++i]);
|
||||
} else if (token == "--timeout" && i + 1 < arg_vec.size()) {
|
||||
timeout_sec = std::stoi(arg_vec[++i]);
|
||||
} else if (absl::StartsWith(token, "--prompt=")) {
|
||||
prompt = token.substr(9);
|
||||
} else if (absl::StartsWith(token, "--host=")) {
|
||||
host = token.substr(7);
|
||||
} else if (absl::StartsWith(token, "--port=")) {
|
||||
port = std::stoi(token.substr(7));
|
||||
} else if (absl::StartsWith(token, "--timeout=")) {
|
||||
timeout_sec = std::stoi(token.substr(10));
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
// Windows doesn't support fork/exec, so users must run tests directly
|
||||
return absl::UnimplementedError(
|
||||
"GUI test command is not supported on Windows. "
|
||||
"Please run yaze_test.exe directly with --enable-ui-tests flag.");
|
||||
#else
|
||||
// Unix-like systems (macOS, Linux) support fork/exec for process spawning
|
||||
std::string test_name = arg_vec[1];
|
||||
if (prompt.empty()) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Usage: agent test --prompt \"<prompt>\" [--host <host>] [--port <port>] [--timeout <sec>]\n\n"
|
||||
"Examples:\n"
|
||||
" z3ed agent test --prompt \"Open Overworld editor\"\n"
|
||||
" z3ed agent test --prompt \"Open Dungeon editor and verify it loads\"\n"
|
||||
" z3ed agent test --prompt \"Click Open ROM button\"");
|
||||
}
|
||||
|
||||
// Get the executable path using platform-specific methods
|
||||
char exe_path[1024];
|
||||
#ifdef __APPLE__
|
||||
uint32_t size = sizeof(exe_path);
|
||||
if (_NSGetExecutablePath(exe_path, &size) != 0) {
|
||||
return absl::InternalError("Could not get executable path");
|
||||
}
|
||||
#elif defined(__linux__)
|
||||
ssize_t len = readlink("/proc/self/exe", exe_path, sizeof(exe_path) - 1);
|
||||
if (len == -1) {
|
||||
return absl::InternalError("Could not get executable path");
|
||||
}
|
||||
exe_path[len] = '\0';
|
||||
#else
|
||||
#ifndef YAZE_WITH_GRPC
|
||||
return absl::UnimplementedError(
|
||||
"GUI test command is not supported on this platform. "
|
||||
"Please run yaze_test directly with --enable-ui-tests flag.");
|
||||
#endif
|
||||
|
||||
// Extract directory from executable path
|
||||
std::string exe_dir = std::string(exe_path);
|
||||
exe_dir = exe_dir.substr(0, exe_dir.find_last_of("/"));
|
||||
std::string yaze_test_path = exe_dir + "/yaze_test";
|
||||
|
||||
// Prepare command arguments for execv
|
||||
std::vector<std::string> command_args;
|
||||
command_args.push_back(yaze_test_path);
|
||||
command_args.push_back("--enable-ui-tests");
|
||||
command_args.push_back("--test=" + test_name);
|
||||
|
||||
std::vector<char*> argv;
|
||||
for (const auto& arg : command_args) {
|
||||
argv.push_back((char*)arg.c_str());
|
||||
"GUI automation requires YAZE_WITH_GRPC=ON at build time.\n"
|
||||
"Rebuild with: cmake -B build -DYAZE_WITH_GRPC=ON");
|
||||
#else
|
||||
std::cout << "\n=== GUI Automation Test ===\n";
|
||||
std::cout << "Prompt: " << prompt << "\n";
|
||||
std::cout << "Server: " << host << ":" << port << "\n\n";
|
||||
|
||||
// Generate workflow from prompt
|
||||
TestWorkflowGenerator generator;
|
||||
auto workflow_or = generator.GenerateWorkflow(prompt);
|
||||
if (!workflow_or.ok()) {
|
||||
return workflow_or.status();
|
||||
}
|
||||
argv.push_back(nullptr);
|
||||
|
||||
// Fork and execute the test process
|
||||
pid_t pid = fork();
|
||||
if (pid == -1) {
|
||||
return absl::InternalError("Failed to fork process");
|
||||
auto workflow = workflow_or.value();
|
||||
|
||||
std::cout << "Generated workflow:\n" << workflow.ToString() << "\n";
|
||||
|
||||
// Connect to test harness
|
||||
GuiAutomationClient client(absl::StrFormat("%s:%d", host, port));
|
||||
auto connect_status = client.Connect();
|
||||
if (!connect_status.ok()) {
|
||||
return absl::UnavailableError(
|
||||
absl::StrFormat(
|
||||
"Failed to connect to test harness at %s:%d\n"
|
||||
"Make sure YAZE is running with:\n"
|
||||
" ./yaze --enable_test_harness --test_harness_port=%d --rom_file=<rom>\n\n"
|
||||
"Error: %s",
|
||||
host, port, port, connect_status.message()));
|
||||
}
|
||||
|
||||
if (pid == 0) {
|
||||
// Child process: execute the test binary
|
||||
execv(yaze_test_path.c_str(), argv.data());
|
||||
// If execv returns, it must have failed
|
||||
_exit(EXIT_FAILURE); // Use _exit in child process after failed exec
|
||||
} else {
|
||||
// Parent process: wait for child to complete
|
||||
int status;
|
||||
if (waitpid(pid, &status, 0) == -1) {
|
||||
return absl::InternalError("Failed to wait for child process");
|
||||
|
||||
std::cout << "✓ Connected to test harness\n\n";
|
||||
|
||||
// Execute workflow
|
||||
auto start_time = std::chrono::steady_clock::now();
|
||||
int step_num = 0;
|
||||
|
||||
for (const auto& step : workflow.steps) {
|
||||
step_num++;
|
||||
std::cout << absl::StrFormat("[%d/%d] %s ... ", step_num,
|
||||
workflow.steps.size(), step.ToString());
|
||||
std::cout.flush();
|
||||
|
||||
absl::StatusOr<AutomationResult> result;
|
||||
|
||||
switch (step.type) {
|
||||
case TestStepType::kClick:
|
||||
result = client.Click(step.target);
|
||||
break;
|
||||
case TestStepType::kType:
|
||||
result = client.Type(step.target, step.text, step.clear_first);
|
||||
break;
|
||||
case TestStepType::kWait:
|
||||
result = client.Wait(step.condition, step.timeout_ms);
|
||||
break;
|
||||
case TestStepType::kAssert:
|
||||
result = client.Assert(step.condition);
|
||||
break;
|
||||
case TestStepType::kScreenshot:
|
||||
result = client.Screenshot();
|
||||
break;
|
||||
}
|
||||
|
||||
if (WIFEXITED(status)) {
|
||||
int exit_code = WEXITSTATUS(status);
|
||||
if (exit_code == 0) {
|
||||
return absl::OkStatus();
|
||||
} else {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("yaze_test exited with code %d", exit_code));
|
||||
}
|
||||
} else if (WIFSIGNALED(status)) {
|
||||
if (!result.ok()) {
|
||||
std::cout << "✗ FAILED\n";
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("yaze_test terminated by signal %d", WTERMSIG(status)));
|
||||
} else {
|
||||
return absl::InternalError("yaze_test terminated abnormally");
|
||||
absl::StrFormat("Step %d failed: %s", step_num,
|
||||
result.status().message()));
|
||||
}
|
||||
|
||||
if (!result->success) {
|
||||
std::cout << "✗ FAILED\n";
|
||||
std::cout << " Error: " << result->message << "\n";
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("Step %d failed: %s", step_num, result->message));
|
||||
}
|
||||
|
||||
std::cout << absl::StrFormat("✓ (%lldms)\n",
|
||||
result->execution_time.count());
|
||||
}
|
||||
|
||||
auto end_time = std::chrono::steady_clock::now();
|
||||
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
end_time - start_time);
|
||||
|
||||
std::cout << "\n✅ Test passed in " << elapsed.count() << "ms\n";
|
||||
return absl::OkStatus();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
251
src/cli/service/gui_automation_client.cc
Normal file
251
src/cli/service/gui_automation_client.cc
Normal file
@@ -0,0 +1,251 @@
|
||||
// gui_automation_client.cc
|
||||
// Implementation of gRPC client for YAZE GUI automation
|
||||
|
||||
#include "cli/service/gui_automation_client.h"
|
||||
|
||||
#include "absl/strings/str_format.h"
|
||||
|
||||
namespace yaze {
|
||||
namespace cli {
|
||||
|
||||
GuiAutomationClient::GuiAutomationClient(const std::string& server_address)
|
||||
: server_address_(server_address) {}
|
||||
|
||||
absl::Status GuiAutomationClient::Connect() {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
auto channel = grpc::CreateChannel(server_address_,
|
||||
grpc::InsecureChannelCredentials());
|
||||
if (!channel) {
|
||||
return absl::InternalError("Failed to create gRPC channel");
|
||||
}
|
||||
|
||||
stub_ = yaze::test::ImGuiTestHarness::NewStub(channel);
|
||||
if (!stub_) {
|
||||
return absl::InternalError("Failed to create gRPC stub");
|
||||
}
|
||||
|
||||
// Test connection with a ping
|
||||
auto result = Ping("connection_test");
|
||||
if (!result.ok()) {
|
||||
return absl::UnavailableError(
|
||||
absl::StrFormat("Failed to connect to test harness at %s: %s",
|
||||
server_address_, result.status().message()));
|
||||
}
|
||||
|
||||
connected_ = true;
|
||||
return absl::OkStatus();
|
||||
#else
|
||||
return absl::UnimplementedError(
|
||||
"GUI automation requires YAZE_WITH_GRPC=ON at build time");
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<AutomationResult> GuiAutomationClient::Ping(
|
||||
const std::string& message) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::PingRequest request;
|
||||
request.set_message(message);
|
||||
|
||||
yaze::test::PingResponse response;
|
||||
grpc::ClientContext context;
|
||||
|
||||
grpc::Status status = stub_->Ping(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("Ping RPC failed: %s", status.error_message()));
|
||||
}
|
||||
|
||||
AutomationResult result;
|
||||
result.success = true;
|
||||
result.message = absl::StrFormat("Server version: %s (timestamp: %s)",
|
||||
response.yaze_version(),
|
||||
response.timestamp_ms());
|
||||
result.execution_time = std::chrono::milliseconds(0);
|
||||
return result;
|
||||
#else
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<AutomationResult> GuiAutomationClient::Click(
|
||||
const std::string& target, ClickType type) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::ClickRequest request;
|
||||
request.set_target(target);
|
||||
|
||||
switch (type) {
|
||||
case ClickType::kLeft:
|
||||
request.set_type(yaze::test::ClickRequest::LEFT);
|
||||
break;
|
||||
case ClickType::kRight:
|
||||
request.set_type(yaze::test::ClickRequest::RIGHT);
|
||||
break;
|
||||
case ClickType::kMiddle:
|
||||
request.set_type(yaze::test::ClickRequest::MIDDLE);
|
||||
break;
|
||||
case ClickType::kDouble:
|
||||
request.set_type(yaze::test::ClickRequest::DOUBLE);
|
||||
break;
|
||||
}
|
||||
|
||||
yaze::test::ClickResponse response;
|
||||
grpc::ClientContext context;
|
||||
|
||||
grpc::Status status = stub_->Click(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("Click RPC failed: %s", status.error_message()));
|
||||
}
|
||||
|
||||
AutomationResult result;
|
||||
result.success = response.success();
|
||||
result.message = response.message();
|
||||
result.execution_time = std::chrono::milliseconds(
|
||||
std::stoll(response.execution_time_ms()));
|
||||
return result;
|
||||
#else
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<AutomationResult> GuiAutomationClient::Type(
|
||||
const std::string& target, const std::string& text, bool clear_first) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::TypeRequest request;
|
||||
request.set_target(target);
|
||||
request.set_text(text);
|
||||
request.set_clear_first(clear_first);
|
||||
|
||||
yaze::test::TypeResponse response;
|
||||
grpc::ClientContext context;
|
||||
|
||||
grpc::Status status = stub_->Type(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("Type RPC failed: %s", status.error_message()));
|
||||
}
|
||||
|
||||
AutomationResult result;
|
||||
result.success = response.success();
|
||||
result.message = response.message();
|
||||
result.execution_time = std::chrono::milliseconds(
|
||||
std::stoll(response.execution_time_ms()));
|
||||
return result;
|
||||
#else
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<AutomationResult> GuiAutomationClient::Wait(
|
||||
const std::string& condition, int timeout_ms, int poll_interval_ms) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::WaitRequest request;
|
||||
request.set_condition(condition);
|
||||
request.set_timeout_ms(timeout_ms);
|
||||
request.set_poll_interval_ms(poll_interval_ms);
|
||||
|
||||
yaze::test::WaitResponse response;
|
||||
grpc::ClientContext context;
|
||||
|
||||
grpc::Status status = stub_->Wait(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("Wait RPC failed: %s", status.error_message()));
|
||||
}
|
||||
|
||||
AutomationResult result;
|
||||
result.success = response.success();
|
||||
result.message = response.message();
|
||||
result.execution_time = std::chrono::milliseconds(
|
||||
std::stoll(response.elapsed_ms()));
|
||||
return result;
|
||||
#else
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<AutomationResult> GuiAutomationClient::Assert(
|
||||
const std::string& condition) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::AssertRequest request;
|
||||
request.set_condition(condition);
|
||||
|
||||
yaze::test::AssertResponse response;
|
||||
grpc::ClientContext context;
|
||||
|
||||
grpc::Status status = stub_->Assert(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("Assert RPC failed: %s", status.error_message()));
|
||||
}
|
||||
|
||||
AutomationResult result;
|
||||
result.success = response.success();
|
||||
result.message = response.message();
|
||||
result.actual_value = response.actual_value();
|
||||
result.expected_value = response.expected_value();
|
||||
result.execution_time = std::chrono::milliseconds(0);
|
||||
return result;
|
||||
#else
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
absl::StatusOr<AutomationResult> GuiAutomationClient::Screenshot(
|
||||
const std::string& region, const std::string& format) {
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
if (!stub_) {
|
||||
return absl::FailedPreconditionError("Not connected. Call Connect() first.");
|
||||
}
|
||||
|
||||
yaze::test::ScreenshotRequest request;
|
||||
request.set_region(region);
|
||||
request.set_format(format);
|
||||
|
||||
yaze::test::ScreenshotResponse response;
|
||||
grpc::ClientContext context;
|
||||
|
||||
grpc::Status status = stub_->Screenshot(&context, request, &response);
|
||||
|
||||
if (!status.ok()) {
|
||||
return absl::InternalError(
|
||||
absl::StrFormat("Screenshot RPC failed: %s", status.error_message()));
|
||||
}
|
||||
|
||||
AutomationResult result;
|
||||
result.success = response.success();
|
||||
result.message = response.message();
|
||||
result.execution_time = std::chrono::milliseconds(0);
|
||||
return result;
|
||||
#else
|
||||
return absl::UnimplementedError("gRPC not available");
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
152
src/cli/service/gui_automation_client.h
Normal file
152
src/cli/service/gui_automation_client.h
Normal file
@@ -0,0 +1,152 @@
|
||||
// gui_automation_client.h
|
||||
// gRPC client for automating YAZE GUI through ImGuiTestHarness service
|
||||
|
||||
#ifndef YAZE_CLI_SERVICE_GUI_AUTOMATION_CLIENT_H
|
||||
#define YAZE_CLI_SERVICE_GUI_AUTOMATION_CLIENT_H
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
#include <grpcpp/grpcpp.h>
|
||||
#include "app/core/proto/imgui_test_harness.grpc.pb.h"
|
||||
#endif
|
||||
|
||||
namespace yaze {
|
||||
namespace cli {
|
||||
|
||||
/**
|
||||
* @brief Type of click action to perform
|
||||
*/
|
||||
enum class ClickType {
|
||||
kLeft,
|
||||
kRight,
|
||||
kMiddle,
|
||||
kDouble
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Result of a GUI automation action
|
||||
*/
|
||||
struct AutomationResult {
|
||||
bool success;
|
||||
std::string message;
|
||||
std::chrono::milliseconds execution_time;
|
||||
std::string actual_value; // For assertions
|
||||
std::string expected_value; // For assertions
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Client for automating YAZE GUI through gRPC
|
||||
*
|
||||
* This client wraps the ImGuiTestHarness gRPC service and provides
|
||||
* a C++ API for CLI commands to drive the YAZE GUI remotely.
|
||||
*
|
||||
* Example usage:
|
||||
* @code
|
||||
* GuiAutomationClient client("localhost:50052");
|
||||
* RETURN_IF_ERROR(client.Connect());
|
||||
*
|
||||
* auto result = client.Click("button:Overworld", ClickType::kLeft);
|
||||
* if (!result.ok()) return result.status();
|
||||
*
|
||||
* if (!result->success) {
|
||||
* return absl::InternalError(result->message);
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
class GuiAutomationClient {
|
||||
public:
|
||||
/**
|
||||
* @brief Construct a new GUI automation client
|
||||
* @param server_address Address of the test harness server (e.g., "localhost:50052")
|
||||
*/
|
||||
explicit GuiAutomationClient(const std::string& server_address);
|
||||
|
||||
/**
|
||||
* @brief Connect to the test harness server
|
||||
* @return Status indicating success or failure
|
||||
*/
|
||||
absl::Status Connect();
|
||||
|
||||
/**
|
||||
* @brief Check if the server is reachable and responsive
|
||||
* @param message Optional message to send in ping
|
||||
* @return Result with server version and timestamp
|
||||
*/
|
||||
absl::StatusOr<AutomationResult> Ping(const std::string& message = "ping");
|
||||
|
||||
/**
|
||||
* @brief Click a GUI element
|
||||
* @param target Target element (format: "button:Label" or "window:Name")
|
||||
* @param type Type of click (left, right, middle, double)
|
||||
* @return Result indicating success/failure and execution time
|
||||
*/
|
||||
absl::StatusOr<AutomationResult> Click(const std::string& target,
|
||||
ClickType type = ClickType::kLeft);
|
||||
|
||||
/**
|
||||
* @brief Type text into an input field
|
||||
* @param target Target input field (format: "input:Label")
|
||||
* @param text Text to type
|
||||
* @param clear_first Whether to clear existing text before typing
|
||||
* @return Result indicating success/failure and execution time
|
||||
*/
|
||||
absl::StatusOr<AutomationResult> Type(const std::string& target,
|
||||
const std::string& text,
|
||||
bool clear_first = false);
|
||||
|
||||
/**
|
||||
* @brief Wait for a condition to be met
|
||||
* @param condition Condition to wait for (e.g., "window_visible:Editor")
|
||||
* @param timeout_ms Maximum time to wait in milliseconds
|
||||
* @param poll_interval_ms How often to check the condition
|
||||
* @return Result indicating whether condition was met
|
||||
*/
|
||||
absl::StatusOr<AutomationResult> Wait(const std::string& condition,
|
||||
int timeout_ms = 5000,
|
||||
int poll_interval_ms = 100);
|
||||
|
||||
/**
|
||||
* @brief Assert a GUI state condition
|
||||
* @param condition Condition to assert (e.g., "visible:Window Name")
|
||||
* @return Result with actual vs expected values
|
||||
*/
|
||||
absl::StatusOr<AutomationResult> Assert(const std::string& condition);
|
||||
|
||||
/**
|
||||
* @brief Capture a screenshot
|
||||
* @param region Region to capture ("full", "window", "element")
|
||||
* @param format Image format ("PNG", "JPEG")
|
||||
* @return Result with file path if successful
|
||||
*/
|
||||
absl::StatusOr<AutomationResult> Screenshot(const std::string& region = "full",
|
||||
const std::string& format = "PNG");
|
||||
|
||||
/**
|
||||
* @brief Check if client is connected
|
||||
*/
|
||||
bool IsConnected() const { return connected_; }
|
||||
|
||||
/**
|
||||
* @brief Get the server address
|
||||
*/
|
||||
const std::string& ServerAddress() const { return server_address_; }
|
||||
|
||||
private:
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
std::unique_ptr<yaze::test::ImGuiTestHarness::Stub> stub_;
|
||||
#endif
|
||||
std::string server_address_;
|
||||
bool connected_ = false;
|
||||
};
|
||||
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
|
||||
#endif // YAZE_CLI_SERVICE_GUI_AUTOMATION_CLIENT_H
|
||||
227
src/cli/service/test_workflow_generator.cc
Normal file
227
src/cli/service/test_workflow_generator.cc
Normal file
@@ -0,0 +1,227 @@
|
||||
// test_workflow_generator.cc
|
||||
// Implementation of natural language to test workflow conversion
|
||||
|
||||
#include "cli/service/test_workflow_generator.h"
|
||||
|
||||
#include "absl/strings/ascii.h"
|
||||
#include "absl/strings/match.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
|
||||
#include <regex>
|
||||
|
||||
namespace yaze {
|
||||
namespace cli {
|
||||
|
||||
std::string TestStep::ToString() const {
|
||||
switch (type) {
|
||||
case TestStepType::kClick:
|
||||
return absl::StrFormat("Click(%s)", target);
|
||||
case TestStepType::kType:
|
||||
return absl::StrFormat("Type(%s, \"%s\"%s)", target, text,
|
||||
clear_first ? ", clear_first" : "");
|
||||
case TestStepType::kWait:
|
||||
return absl::StrFormat("Wait(%s, %dms)", condition, timeout_ms);
|
||||
case TestStepType::kAssert:
|
||||
return absl::StrFormat("Assert(%s)", condition);
|
||||
case TestStepType::kScreenshot:
|
||||
return "Screenshot()";
|
||||
}
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
std::string TestWorkflow::ToString() const {
|
||||
std::string result = absl::StrCat("Workflow: ", description, "\n");
|
||||
for (size_t i = 0; i < steps.size(); ++i) {
|
||||
absl::StrAppend(&result, " ", i + 1, ". ", steps[i].ToString(), "\n");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
absl::StatusOr<TestWorkflow> TestWorkflowGenerator::GenerateWorkflow(
|
||||
const std::string& prompt) {
|
||||
std::string normalized_prompt = absl::AsciiStrToLower(prompt);
|
||||
|
||||
// Try pattern matching in order of specificity
|
||||
std::string editor_name, input_name, text, button_name;
|
||||
|
||||
// Pattern 1: "Open <Editor> and verify it loads"
|
||||
if (MatchesOpenAndVerify(normalized_prompt, &editor_name)) {
|
||||
return BuildOpenAndVerifyWorkflow(editor_name);
|
||||
}
|
||||
|
||||
// Pattern 2: "Open <Editor> editor"
|
||||
if (MatchesOpenEditor(normalized_prompt, &editor_name)) {
|
||||
return BuildOpenEditorWorkflow(editor_name);
|
||||
}
|
||||
|
||||
// Pattern 3: "Type '<text>' in <input>"
|
||||
if (MatchesTypeInput(normalized_prompt, &input_name, &text)) {
|
||||
return BuildTypeInputWorkflow(input_name, text);
|
||||
}
|
||||
|
||||
// Pattern 4: "Click <button>"
|
||||
if (MatchesClickButton(normalized_prompt, &button_name)) {
|
||||
return BuildClickButtonWorkflow(button_name);
|
||||
}
|
||||
|
||||
// If no patterns match, return helpful error
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrFormat(
|
||||
"Unable to parse prompt: \"%s\"\n\n"
|
||||
"Supported patterns:\n"
|
||||
" - Open <Editor> editor\n"
|
||||
" - Open <Editor> and verify it loads\n"
|
||||
" - Type '<text>' in <input>\n"
|
||||
" - Click <button>\n\n"
|
||||
"Examples:\n"
|
||||
" - Open Overworld editor\n"
|
||||
" - Open Dungeon editor and verify it loads\n"
|
||||
" - Type 'zelda3.sfc' in filename input\n"
|
||||
" - Click Open ROM button",
|
||||
prompt));
|
||||
}
|
||||
|
||||
bool TestWorkflowGenerator::MatchesOpenEditor(const std::string& prompt,
|
||||
std::string* editor_name) {
|
||||
// Match: "open <name> editor" or "open <name>"
|
||||
std::regex pattern(R"(open\s+(\w+)(?:\s+editor)?)");
|
||||
std::smatch match;
|
||||
if (std::regex_search(prompt, match, pattern) && match.size() > 1) {
|
||||
*editor_name = match[1].str();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool TestWorkflowGenerator::MatchesOpenAndVerify(const std::string& prompt,
|
||||
std::string* editor_name) {
|
||||
// Match: "open <name> and verify" or "open <name> editor and verify it loads"
|
||||
std::regex pattern(R"(open\s+(\w+)(?:\s+editor)?\s+and\s+verify)");
|
||||
std::smatch match;
|
||||
if (std::regex_search(prompt, match, pattern) && match.size() > 1) {
|
||||
*editor_name = match[1].str();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool TestWorkflowGenerator::MatchesTypeInput(const std::string& prompt,
|
||||
std::string* input_name,
|
||||
std::string* text) {
|
||||
// Match: "type 'text' in <input>" or "type \"text\" in <input>"
|
||||
std::regex pattern(R"(type\s+['"]([^'"]+)['"]\s+in(?:to)?\s+(\w+))");
|
||||
std::smatch match;
|
||||
if (std::regex_search(prompt, match, pattern) && match.size() > 2) {
|
||||
*text = match[1].str();
|
||||
*input_name = match[2].str();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool TestWorkflowGenerator::MatchesClickButton(const std::string& prompt,
|
||||
std::string* button_name) {
|
||||
// Match: "click <button>" or "click <button> button"
|
||||
std::regex pattern(R"(click\s+([\w\s]+?)(?:\s+button)?\s*$)");
|
||||
std::smatch match;
|
||||
if (std::regex_search(prompt, match, pattern) && match.size() > 1) {
|
||||
*button_name = match[1].str();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string TestWorkflowGenerator::NormalizeEditorName(const std::string& name) {
|
||||
std::string normalized = name;
|
||||
// Capitalize first letter
|
||||
if (!normalized.empty()) {
|
||||
normalized[0] = std::toupper(normalized[0]);
|
||||
}
|
||||
// Add " Editor" suffix if not present
|
||||
if (!absl::StrContains(absl::AsciiStrToLower(normalized), "editor")) {
|
||||
absl::StrAppend(&normalized, " Editor");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
TestWorkflow TestWorkflowGenerator::BuildOpenEditorWorkflow(
|
||||
const std::string& editor_name) {
|
||||
std::string normalized_name = NormalizeEditorName(editor_name);
|
||||
|
||||
TestWorkflow workflow;
|
||||
workflow.description = absl::StrFormat("Open %s", normalized_name);
|
||||
|
||||
// Step 1: Click the editor button
|
||||
TestStep click_step;
|
||||
click_step.type = TestStepType::kClick;
|
||||
click_step.target = absl::StrFormat("button:%s",
|
||||
absl::StrReplaceAll(normalized_name,
|
||||
{{" Editor", ""}}));
|
||||
workflow.steps.push_back(click_step);
|
||||
|
||||
// Step 2: Wait for editor window to appear
|
||||
TestStep wait_step;
|
||||
wait_step.type = TestStepType::kWait;
|
||||
wait_step.condition = absl::StrFormat("window_visible:%s", normalized_name);
|
||||
wait_step.timeout_ms = 5000;
|
||||
workflow.steps.push_back(wait_step);
|
||||
|
||||
return workflow;
|
||||
}
|
||||
|
||||
TestWorkflow TestWorkflowGenerator::BuildOpenAndVerifyWorkflow(
|
||||
const std::string& editor_name) {
|
||||
// Start with basic open workflow
|
||||
TestWorkflow workflow = BuildOpenEditorWorkflow(editor_name);
|
||||
workflow.description = absl::StrFormat("Open and verify %s",
|
||||
NormalizeEditorName(editor_name));
|
||||
|
||||
// Add assertion step
|
||||
TestStep assert_step;
|
||||
assert_step.type = TestStepType::kAssert;
|
||||
assert_step.condition = absl::StrFormat("visible:%s",
|
||||
NormalizeEditorName(editor_name));
|
||||
workflow.steps.push_back(assert_step);
|
||||
|
||||
return workflow;
|
||||
}
|
||||
|
||||
TestWorkflow TestWorkflowGenerator::BuildTypeInputWorkflow(
|
||||
const std::string& input_name, const std::string& text) {
|
||||
TestWorkflow workflow;
|
||||
workflow.description = absl::StrFormat("Type '%s' into %s", text, input_name);
|
||||
|
||||
// Step 1: Click input to focus
|
||||
TestStep click_step;
|
||||
click_step.type = TestStepType::kClick;
|
||||
click_step.target = absl::StrFormat("input:%s", input_name);
|
||||
workflow.steps.push_back(click_step);
|
||||
|
||||
// Step 2: Type the text
|
||||
TestStep type_step;
|
||||
type_step.type = TestStepType::kType;
|
||||
type_step.target = absl::StrFormat("input:%s", input_name);
|
||||
type_step.text = text;
|
||||
type_step.clear_first = true;
|
||||
workflow.steps.push_back(type_step);
|
||||
|
||||
return workflow;
|
||||
}
|
||||
|
||||
TestWorkflow TestWorkflowGenerator::BuildClickButtonWorkflow(
|
||||
const std::string& button_name) {
|
||||
TestWorkflow workflow;
|
||||
workflow.description = absl::StrFormat("Click '%s' button", button_name);
|
||||
|
||||
TestStep click_step;
|
||||
click_step.type = TestStepType::kClick;
|
||||
click_step.target = absl::StrFormat("button:%s", button_name);
|
||||
workflow.steps.push_back(click_step);
|
||||
|
||||
return workflow;
|
||||
}
|
||||
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
106
src/cli/service/test_workflow_generator.h
Normal file
106
src/cli/service/test_workflow_generator.h
Normal file
@@ -0,0 +1,106 @@
|
||||
// test_workflow_generator.h
|
||||
// Converts natural language prompts into GUI automation workflows
|
||||
|
||||
#ifndef YAZE_CLI_SERVICE_TEST_WORKFLOW_GENERATOR_H
|
||||
#define YAZE_CLI_SERVICE_TEST_WORKFLOW_GENERATOR_H
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace yaze {
|
||||
namespace cli {
|
||||
|
||||
/**
|
||||
* @brief Type of test step to execute
|
||||
*/
|
||||
enum class TestStepType {
|
||||
kClick, // Click a button or element
|
||||
kType, // Type text into an input
|
||||
kWait, // Wait for a condition
|
||||
kAssert, // Assert a condition is true
|
||||
kScreenshot // Capture a screenshot
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A single step in a GUI test workflow
|
||||
*/
|
||||
struct TestStep {
|
||||
TestStepType type;
|
||||
std::string target; // Widget/element target (e.g., "button:Overworld")
|
||||
std::string text; // Text to type (for kType steps)
|
||||
std::string condition; // Condition to wait for or assert
|
||||
int timeout_ms = 5000; // Timeout for wait operations
|
||||
bool clear_first = false; // Clear text before typing
|
||||
|
||||
std::string ToString() const;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A complete GUI test workflow
|
||||
*/
|
||||
struct TestWorkflow {
|
||||
std::string description;
|
||||
std::vector<TestStep> steps;
|
||||
|
||||
std::string ToString() const;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Generates GUI test workflows from natural language prompts
|
||||
*
|
||||
* This class uses pattern matching to convert user prompts into
|
||||
* structured test workflows that can be executed by GuiAutomationClient.
|
||||
*
|
||||
* Example prompts:
|
||||
* - "Open Overworld editor" → Click button, Wait for window
|
||||
* - "Open Dungeon editor and verify it loads" → Click, Wait, Assert
|
||||
* - "Type 'zelda3.sfc' in filename input" → Click input, Type text
|
||||
*
|
||||
* Usage:
|
||||
* @code
|
||||
* TestWorkflowGenerator generator;
|
||||
* auto workflow = generator.GenerateWorkflow("Open Overworld editor");
|
||||
* if (!workflow.ok()) return workflow.status();
|
||||
*
|
||||
* for (const auto& step : workflow->steps) {
|
||||
* std::cout << step.ToString() << "\n";
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
class TestWorkflowGenerator {
|
||||
public:
|
||||
TestWorkflowGenerator() = default;
|
||||
|
||||
/**
|
||||
* @brief Generate a test workflow from a natural language prompt
|
||||
* @param prompt Natural language description of desired GUI actions
|
||||
* @return TestWorkflow or error if prompt is unsupported
|
||||
*/
|
||||
absl::StatusOr<TestWorkflow> GenerateWorkflow(const std::string& prompt);
|
||||
|
||||
private:
|
||||
// Pattern matchers for different prompt types
|
||||
bool MatchesOpenEditor(const std::string& prompt, std::string* editor_name);
|
||||
bool MatchesOpenAndVerify(const std::string& prompt, std::string* editor_name);
|
||||
bool MatchesTypeInput(const std::string& prompt, std::string* input_name,
|
||||
std::string* text);
|
||||
bool MatchesClickButton(const std::string& prompt, std::string* button_name);
|
||||
bool MatchesMultiStep(const std::string& prompt);
|
||||
|
||||
// Workflow builders
|
||||
TestWorkflow BuildOpenEditorWorkflow(const std::string& editor_name);
|
||||
TestWorkflow BuildOpenAndVerifyWorkflow(const std::string& editor_name);
|
||||
TestWorkflow BuildTypeInputWorkflow(const std::string& input_name,
|
||||
const std::string& text);
|
||||
TestWorkflow BuildClickButtonWorkflow(const std::string& button_name);
|
||||
|
||||
// Helper to normalize editor names (e.g., "overworld" → "Overworld Editor")
|
||||
std::string NormalizeEditorName(const std::string& name);
|
||||
};
|
||||
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
|
||||
#endif // YAZE_CLI_SERVICE_TEST_WORKFLOW_GENERATOR_H
|
||||
Reference in New Issue
Block a user