From 9d5919adb5ea5b869a657ca4a897ce424d85563f Mon Sep 17 00:00:00 2001 From: scawful Date: Sat, 4 Oct 2025 22:19:09 -0400 Subject: [PATCH] feat: Add AI action parser for natural language command processing - Introduced `AIActionParser` class to parse natural language commands into structured GUI actions, supporting commands like placing tiles and opening editors. - Implemented helper functions for extracting coordinates and parsing hex/decimal values. - Added action types for various AI actions, including selecting and placing tiles, saving changes, and clicking buttons. - Created header file `ai_action_parser.h` to define the action types and parser interface. - Added implementation file `ai_action_parser.cc` with command parsing logic and pattern matching for different action types. --- .../service/agent/learned_knowledge_service.h | 2 +- src/cli/service/ai/ai_action_parser.cc | 273 ++++++++++++++++++ src/cli/service/ai/ai_action_parser.h | 86 ++++++ 3 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 src/cli/service/ai/ai_action_parser.cc create mode 100644 src/cli/service/ai/ai_action_parser.h diff --git a/src/cli/service/agent/learned_knowledge_service.h b/src/cli/service/agent/learned_knowledge_service.h index a03bd67a..ba01c19a 100644 --- a/src/cli/service/agent/learned_knowledge_service.h +++ b/src/cli/service/agent/learned_knowledge_service.h @@ -213,7 +213,7 @@ class LearnedKnowledgeService { absl::Status SavePreferences(); absl::Status SavePatterns(); absl::Status SaveProjects(); - absl:Status SaveMemories(); + absl::Status SaveMemories(); std::string GenerateID() const; }; diff --git a/src/cli/service/ai/ai_action_parser.cc b/src/cli/service/ai/ai_action_parser.cc new file mode 100644 index 00000000..e6c01778 --- /dev/null +++ b/src/cli/service/ai/ai_action_parser.cc @@ -0,0 +1,273 @@ +#include "cli/service/ai/ai_action_parser.h" + +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" + +namespace yaze { +namespace cli { +namespace ai { + +namespace { + +// Helper to convert hex string to int +int ParseHexOrDecimal(const std::string& str) { + if (absl::StartsWith(str, "0x") || absl::StartsWith(str, "0X")) { + return std::stoi(str, nullptr, 16); + } + return std::stoi(str); +} + +// Helper to extract coordinates like "(5, 7)" or "5,7" or "x=5 y=7" +bool ExtractCoordinates(const std::string& text, int* x, int* y) { + // Pattern: (X, Y) or X,Y or x=X y=Y + std::regex coord_pattern(R"(\(?(\d+)\s*,\s*(\d+)\)?)"); + std::smatch match; + + if (std::regex_search(text, match, coord_pattern) && match.size() >= 3) { + *x = std::stoi(match[1].str()); + *y = std::stoi(match[2].str()); + return true; + } + + // Try x=X y=Y format + std::regex xy_pattern(R"(x\s*=\s*(\d+).*y\s*=\s*(\d+))", std::regex::icase); + if (std::regex_search(text, match, xy_pattern) && match.size() >= 3) { + *x = std::stoi(match[1].str()); + *y = std::stoi(match[2].str()); + return true; + } + + return false; +} + +} // namespace + +absl::StatusOr> AIActionParser::ParseCommand( + const std::string& command) { + std::vector actions; + + std::string cmd_lower = command; + std::transform(cmd_lower.begin(), cmd_lower.end(), cmd_lower.begin(), ::tolower); + + // Try to match different patterns + std::map params; + + // Pattern 1: "Place tile X at position (Y, Z)" + if (MatchesPlaceTilePattern(command, ¶ms)) { + // Actions: Select tile, place tile + actions.push_back(AIAction(AIActionType::kSelectTile, params)); + actions.push_back(AIAction(AIActionType::kPlaceTile, params)); + actions.push_back(AIAction(AIActionType::kSaveTile, {})); + return actions; + } + + // Pattern 2: "Select tile X" + if (MatchesSelectTilePattern(command, ¶ms)) { + actions.push_back(AIAction(AIActionType::kSelectTile, params)); + return actions; + } + + // Pattern 3: "Open overworld editor" + if (MatchesOpenEditorPattern(command, ¶ms)) { + actions.push_back(AIAction(AIActionType::kOpenEditor, params)); + return actions; + } + + // Pattern 4: Simple button clicks + if (absl::StrContains(cmd_lower, "click") || absl::StrContains(cmd_lower, "press")) { + std::regex button_pattern(R"((save|load|export|import|open)\s+(\w+))", std::regex::icase); + std::smatch match; + if (std::regex_search(command, match, button_pattern)) { + params["button"] = match[1].str() + " " + match[2].str(); + actions.push_back(AIAction(AIActionType::kClickButton, params)); + return actions; + } + } + + return absl::InvalidArgumentError( + absl::StrCat("Could not parse AI command: ", command)); +} + +std::string AIActionParser::ActionToString(const AIAction& action) { + switch (action.type) { + case AIActionType::kOpenEditor: { + auto it = action.parameters.find("editor"); + if (it != action.parameters.end()) { + return absl::StrCat("Open ", it->second, " editor"); + } + return "Open editor"; + } + + case AIActionType::kSelectTile: { + auto it = action.parameters.find("tile_id"); + if (it != action.parameters.end()) { + return absl::StrCat("Select tile ", it->second); + } + return "Select tile"; + } + + case AIActionType::kPlaceTile: { + auto x_it = action.parameters.find("x"); + auto y_it = action.parameters.find("y"); + if (x_it != action.parameters.end() && y_it != action.parameters.end()) { + return absl::StrCat("Place tile at position (", x_it->second, ", ", y_it->second, ")"); + } + return "Place tile"; + } + + case AIActionType::kSaveTile: + return "Save changes to ROM"; + + case AIActionType::kVerifyTile: + return "Verify tile placement"; + + case AIActionType::kClickButton: { + auto it = action.parameters.find("button"); + if (it != action.parameters.end()) { + return absl::StrCat("Click ", it->second, " button"); + } + return "Click button"; + } + + case AIActionType::kWait: + return "Wait"; + + case AIActionType::kScreenshot: + return "Take screenshot"; + + case AIActionType::kInvalidAction: + return "Invalid action"; + } + + return "Unknown action"; +} + +bool AIActionParser::MatchesPlaceTilePattern( + const std::string& command, + std::map* params) { + std::string cmd_lower = command; + std::transform(cmd_lower.begin(), cmd_lower.end(), cmd_lower.begin(), ::tolower); + + if (!absl::StrContains(cmd_lower, "place") && + !absl::StrContains(cmd_lower, "put") && + !absl::StrContains(cmd_lower, "set")) { + return false; + } + + if (!absl::StrContains(cmd_lower, "tile")) { + return false; + } + + // Extract tile ID + std::regex tile_pattern(R"(tile\s+(?:id\s+)?(0x[0-9a-fA-F]+|\d+))", std::regex::icase); + std::smatch match; + + if (std::regex_search(command, match, tile_pattern)) { + try { + int tile_id = ParseHexOrDecimal(match[1].str()); + (*params)["tile_id"] = std::to_string(tile_id); + } catch (...) { + return false; + } + } else { + return false; + } + + // Extract coordinates + int x, y; + if (ExtractCoordinates(command, &x, &y)) { + (*params)["x"] = std::to_string(x); + (*params)["y"] = std::to_string(y); + } else { + return false; + } + + // Extract map ID if specified + std::regex map_pattern(R"((?:map|overworld)\s+(?:id\s+)?(\d+))", std::regex::icase); + if (std::regex_search(command, match, map_pattern)) { + (*params)["map_id"] = match[1].str(); + } else { + (*params)["map_id"] = "0"; // Default to map 0 + } + + return true; +} + +bool AIActionParser::MatchesSelectTilePattern( + const std::string& command, + std::map* params) { + std::string cmd_lower = command; + std::transform(cmd_lower.begin(), cmd_lower.end(), cmd_lower.begin(), ::tolower); + + if (!absl::StrContains(cmd_lower, "select") && + !absl::StrContains(cmd_lower, "choose") && + !absl::StrContains(cmd_lower, "pick")) { + return false; + } + + if (!absl::StrContains(cmd_lower, "tile")) { + return false; + } + + // Extract tile ID + std::regex tile_pattern(R"(tile\s+(?:id\s+)?(0x[0-9a-fA-F]+|\d+))", std::regex::icase); + std::smatch match; + + if (std::regex_search(command, match, tile_pattern)) { + try { + int tile_id = ParseHexOrDecimal(match[1].str()); + (*params)["tile_id"] = std::to_string(tile_id); + return true; + } catch (...) { + return false; + } + } + + return false; +} + +bool AIActionParser::MatchesOpenEditorPattern( + const std::string& command, + std::map* params) { + std::string cmd_lower = command; + std::transform(cmd_lower.begin(), cmd_lower.end(), cmd_lower.begin(), ::tolower); + + if (!absl::StrContains(cmd_lower, "open") && + !absl::StrContains(cmd_lower, "launch") && + !absl::StrContains(cmd_lower, "start")) { + return false; + } + + if (absl::StrContains(cmd_lower, "overworld")) { + (*params)["editor"] = "overworld"; + return true; + } + + if (absl::StrContains(cmd_lower, "dungeon")) { + (*params)["editor"] = "dungeon"; + return true; + } + + if (absl::StrContains(cmd_lower, "sprite")) { + (*params)["editor"] = "sprite"; + return true; + } + + if (absl::StrContains(cmd_lower, "tile16") || absl::StrContains(cmd_lower, "tile 16")) { + (*params)["editor"] = "tile16"; + return true; + } + + return false; +} + +} // namespace ai +} // namespace cli +} // namespace yaze diff --git a/src/cli/service/ai/ai_action_parser.h b/src/cli/service/ai/ai_action_parser.h new file mode 100644 index 00000000..062cfd12 --- /dev/null +++ b/src/cli/service/ai/ai_action_parser.h @@ -0,0 +1,86 @@ +#ifndef YAZE_CLI_SERVICE_AI_AI_ACTION_PARSER_H_ +#define YAZE_CLI_SERVICE_AI_AI_ACTION_PARSER_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" + +namespace yaze { +namespace cli { +namespace ai { + +/** + * @enum AIActionType + * @brief Types of actions the AI can request + */ +enum class AIActionType { + kOpenEditor, // Open a specific editor window + kSelectTile, // Select a tile from the tile16 selector + kPlaceTile, // Place a tile at a specific position + kSaveTile, // Save tile changes to ROM + kVerifyTile, // Verify a tile was placed correctly + kClickButton, // Click a specific button + kWait, // Wait for a duration or condition + kScreenshot, // Take a screenshot for verification + kInvalidAction +}; + +/** + * @struct AIAction + * @brief Represents a single action to be performed in the GUI + */ +struct AIAction { + AIActionType type; + std::map parameters; + + AIAction() : type(AIActionType::kInvalidAction) {} + AIAction(AIActionType t) : type(t) {} + AIAction(AIActionType t, const std::map& params) + : type(t), parameters(params) {} +}; + +/** + * @class AIActionParser + * @brief Parses natural language commands into structured GUI actions + * + * Understands commands like: + * - "Place tile 0x42 at overworld position (5, 7)" + * - "Open the overworld editor" + * - "Select tile 100 from the tile selector" + */ +class AIActionParser { + public: + /** + * Parse a natural language command into a sequence of AI actions + * @param command The command to parse + * @return Vector of actions, or error status + */ + static absl::StatusOr> ParseCommand( + const std::string& command); + + /** + * Convert an action back to a human-readable string + */ + static std::string ActionToString(const AIAction& action); + + private: + static AIActionType ParseActionType(const std::string& verb); + static std::map ExtractParameters( + const std::string& command, AIActionType type); + + // Pattern matchers for different command types + static bool MatchesPlaceTilePattern(const std::string& command, + std::map* params); + static bool MatchesSelectTilePattern(const std::string& command, + std::map* params); + static bool MatchesOpenEditorPattern(const std::string& command, + std::map* params); +}; + +} // namespace ai +} // namespace cli +} // namespace yaze + +#endif // YAZE_CLI_SERVICE_AI_AI_ACTION_PARSER_H_