From 7e2f0454d316b262b090ec354b02480ae3d1ccb1 Mon Sep 17 00:00:00 2001 From: scawful Date: Sun, 5 Oct 2025 23:42:55 -0400 Subject: [PATCH] feat: Implement Canvas Automation Service for Enhanced GUI Control - Introduced a new gRPC service for canvas automation, enabling remote control of canvas operations for AI agents, GUI testing, and collaborative editing. - Added proto definitions for canvas automation, including tile operations, selection management, and view control functionalities. - Implemented the CanvasAutomationServiceImpl class to handle various canvas operations, ensuring a robust interface for automation tasks. - Updated CMake configuration to include new proto files and service implementations, enhancing the build system for the canvas automation features. --- src/app/core/core_library.cmake | 6 +- .../core/service/canvas_automation_service.cc | 338 ++++++++++++++++++ .../core/service/canvas_automation_service.h | 133 +++++++ src/protos/canvas_automation.proto | 236 ++++++++++++ .../proto => protos}/imgui_test_harness.proto | 0 5 files changed, 711 insertions(+), 2 deletions(-) create mode 100644 src/app/core/service/canvas_automation_service.cc create mode 100644 src/app/core/service/canvas_automation_service.h create mode 100644 src/protos/canvas_automation.proto rename src/{app/core/proto => protos}/imgui_test_harness.proto (100%) diff --git a/src/app/core/core_library.cmake b/src/app/core/core_library.cmake index 08ddd8c2..c6aae590 100644 --- a/src/app/core/core_library.cmake +++ b/src/app/core/core_library.cmake @@ -83,11 +83,13 @@ if(YAZE_WITH_GRPC) ${CMAKE_SOURCE_DIR}/third_party/json/include) target_compile_definitions(yaze_core_lib PRIVATE YAZE_WITH_JSON) - # Add proto definitions for test harness and ROM service + # Add proto definitions for test harness, ROM service, and canvas automation target_add_protobuf(yaze_core_lib - ${PROJECT_SOURCE_DIR}/src/app/core/proto/imgui_test_harness.proto) + ${PROJECT_SOURCE_DIR}/src/protos/imgui_test_harness.proto) target_add_protobuf(yaze_core_lib ${PROJECT_SOURCE_DIR}/src/protos/rom_service.proto) + target_add_protobuf(yaze_core_lib + ${PROJECT_SOURCE_DIR}/src/protos/canvas_automation.proto) # Add test harness implementation target_sources(yaze_core_lib PRIVATE diff --git a/src/app/core/service/canvas_automation_service.cc b/src/app/core/service/canvas_automation_service.cc new file mode 100644 index 00000000..5a3fd318 --- /dev/null +++ b/src/app/core/service/canvas_automation_service.cc @@ -0,0 +1,338 @@ +#include "app/core/service/canvas_automation_service.h" + +#ifdef YAZE_WITH_GRPC + +#include "src/protos/canvas_automation.pb.h" +#include "app/editor/overworld/overworld_editor.h" +#include "app/gui/canvas/canvas_automation_api.h" + +namespace yaze { + +void CanvasAutomationServiceImpl::RegisterCanvas(const std::string& canvas_id, + gui::Canvas* canvas) { + canvases_[canvas_id] = canvas; +} + +void CanvasAutomationServiceImpl::RegisterOverworldEditor( + const std::string& canvas_id, editor::OverworldEditor* editor) { + overworld_editors_[canvas_id] = editor; +} + +gui::Canvas* CanvasAutomationServiceImpl::GetCanvas(const std::string& canvas_id) { + auto it = canvases_.find(canvas_id); + if (it != canvases_.end()) { + return it->second; + } + return nullptr; +} + +editor::OverworldEditor* CanvasAutomationServiceImpl::GetOverworldEditor( + const std::string& canvas_id) { + auto it = overworld_editors_.find(canvas_id); + if (it != overworld_editors_.end()) { + return it->second; + } + return nullptr; +} + +// ============================================================================ +// Tile Operations +// ============================================================================ + +absl::Status CanvasAutomationServiceImpl::SetTile( + const proto::SetTileRequest* request, proto::SetTileResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + bool success = api->SetTileAt(request->x(), request->y(), request->tile_id()); + + response->set_success(success); + if (!success) { + response->set_error("Failed to set tile - out of bounds or callback failed"); + } + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::GetTile( + const proto::GetTileRequest* request, proto::GetTileResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + int tile_id = api->GetTileAt(request->x(), request->y()); + + if (tile_id >= 0) { + response->set_tile_id(tile_id); + response->set_success(true); + } else { + response->set_success(false); + response->set_error("Tile not found - out of bounds or no callback set"); + } + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::SetTiles( + const proto::SetTilesRequest* request, proto::SetTilesResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + + std::vector> tiles; + for (const auto& tile : request->tiles()) { + tiles.push_back({tile.x(), tile.y(), tile.tile_id()}); + } + + bool success = api->SetTiles(tiles); + response->set_success(success); + response->set_tiles_painted(tiles.size()); + + return absl::OkStatus(); +} + +// ============================================================================ +// Selection Operations +// ============================================================================ + +absl::Status CanvasAutomationServiceImpl::SelectTile( + const proto::SelectTileRequest* request, proto::SelectTileResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + api->SelectTile(request->x(), request->y()); + response->set_success(true); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::SelectTileRect( + const proto::SelectTileRectRequest* request, + proto::SelectTileRectResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + const auto& rect = request->rect(); + api->SelectTileRect(rect.x1(), rect.y1(), rect.x2(), rect.y2()); + + auto selection = api->GetSelection(); + response->set_success(true); + response->set_tiles_selected(selection.selected_tiles.size()); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::GetSelection( + const proto::GetSelectionRequest* request, + proto::GetSelectionResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + return absl::NotFoundError("Canvas not found: " + request->canvas_id()); + } + + auto* api = canvas->GetAutomationAPI(); + auto selection = api->GetSelection(); + + response->set_has_selection(selection.has_selection); + + for (const auto& tile : selection.selected_tiles) { + auto* coord = response->add_selected_tiles(); + coord->set_x(static_cast(tile.x)); + coord->set_y(static_cast(tile.y)); + } + + auto* start = response->mutable_selection_start(); + start->set_x(static_cast(selection.selection_start.x)); + start->set_y(static_cast(selection.selection_start.y)); + + auto* end = response->mutable_selection_end(); + end->set_x(static_cast(selection.selection_end.x)); + end->set_y(static_cast(selection.selection_end.y)); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::ClearSelection( + const proto::ClearSelectionRequest* request, + proto::ClearSelectionResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + return absl::NotFoundError("Canvas not found: " + request->canvas_id()); + } + + auto* api = canvas->GetAutomationAPI(); + api->ClearSelection(); + response->set_success(true); + + return absl::OkStatus(); +} + +// ============================================================================ +// View Operations +// ============================================================================ + +absl::Status CanvasAutomationServiceImpl::ScrollToTile( + const proto::ScrollToTileRequest* request, + proto::ScrollToTileResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + api->ScrollToTile(request->x(), request->y(), request->center()); + response->set_success(true); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::CenterOn( + const proto::CenterOnRequest* request, proto::CenterOnResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + api->CenterOn(request->x(), request->y()); + response->set_success(true); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::SetZoom( + const proto::SetZoomRequest* request, proto::SetZoomResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + response->set_success(false); + response->set_error("Canvas not found: " + request->canvas_id()); + return absl::NotFoundError(response->error()); + } + + auto* api = canvas->GetAutomationAPI(); + api->SetZoom(request->zoom()); + + float actual_zoom = api->GetZoom(); + response->set_success(true); + response->set_actual_zoom(actual_zoom); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::GetZoom( + const proto::GetZoomRequest* request, proto::GetZoomResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + return absl::NotFoundError("Canvas not found: " + request->canvas_id()); + } + + auto* api = canvas->GetAutomationAPI(); + response->set_zoom(api->GetZoom()); + + return absl::OkStatus(); +} + +// ============================================================================ +// Query Operations +// ============================================================================ + +absl::Status CanvasAutomationServiceImpl::GetDimensions( + const proto::GetDimensionsRequest* request, + proto::GetDimensionsResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + return absl::NotFoundError("Canvas not found: " + request->canvas_id()); + } + + auto* api = canvas->GetAutomationAPI(); + auto dims = api->GetDimensions(); + + auto* proto_dims = response->mutable_dimensions(); + proto_dims->set_width_tiles(dims.width_tiles); + proto_dims->set_height_tiles(dims.height_tiles); + proto_dims->set_tile_size(dims.tile_size); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::GetVisibleRegion( + const proto::GetVisibleRegionRequest* request, + proto::GetVisibleRegionResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + return absl::NotFoundError("Canvas not found: " + request->canvas_id()); + } + + auto* api = canvas->GetAutomationAPI(); + auto region = api->GetVisibleRegion(); + + auto* proto_region = response->mutable_region(); + proto_region->set_min_x(region.min_x); + proto_region->set_min_y(region.min_y); + proto_region->set_max_x(region.max_x); + proto_region->set_max_y(region.max_y); + + return absl::OkStatus(); +} + +absl::Status CanvasAutomationServiceImpl::IsTileVisible( + const proto::IsTileVisibleRequest* request, + proto::IsTileVisibleResponse* response) { + + auto* canvas = GetCanvas(request->canvas_id()); + if (!canvas) { + return absl::NotFoundError("Canvas not found: " + request->canvas_id()); + } + + auto* api = canvas->GetAutomationAPI(); + response->set_is_visible(api->IsTileVisible(request->x(), request->y())); + + return absl::OkStatus(); +} + +} // namespace yaze + +#endif // YAZE_WITH_GRPC + diff --git a/src/app/core/service/canvas_automation_service.h b/src/app/core/service/canvas_automation_service.h new file mode 100644 index 00000000..ebf30ec4 --- /dev/null +++ b/src/app/core/service/canvas_automation_service.h @@ -0,0 +1,133 @@ +#ifndef YAZE_APP_CORE_SERVICE_CANVAS_AUTOMATION_SERVICE_H_ +#define YAZE_APP_CORE_SERVICE_CANVAS_AUTOMATION_SERVICE_H_ + +#ifdef YAZE_WITH_GRPC + +#include +#include +#include + +#include "absl/status/status.h" +#include "app/gui/canvas.h" + +// Forward declarations +namespace grpc { +class ServerContext; +} + +namespace yaze { +namespace editor { +class OverworldEditor; +} + +namespace proto { +// Forward declare proto types +class SetTileRequest; +class SetTileResponse; +class GetTileRequest; +class GetTileResponse; +class SetTilesRequest; +class SetTilesResponse; +class SelectTileRequest; +class SelectTileResponse; +class SelectTileRectRequest; +class SelectTileRectResponse; +class GetSelectionRequest; +class GetSelectionResponse; +class ClearSelectionRequest; +class ClearSelectionResponse; +class ScrollToTileRequest; +class ScrollToTileResponse; +class CenterOnRequest; +class CenterOnResponse; +class SetZoomRequest; +class SetZoomResponse; +class GetZoomRequest; +class GetZoomResponse; +class GetDimensionsRequest; +class GetDimensionsResponse; +class GetVisibleRegionRequest; +class GetVisibleRegionResponse; +class IsTileVisibleRequest; +class IsTileVisibleResponse; +} // namespace proto + +/** + * @brief Implementation of CanvasAutomation gRPC service + * + * Provides remote access to canvas automation API for: + * - AI agent tool calls + * - Remote GUI testing + * - Collaborative editing workflows + * - CLI automation scripts + */ +class CanvasAutomationServiceImpl { + public: + CanvasAutomationServiceImpl() = default; + + // Register a canvas for automation + void RegisterCanvas(const std::string& canvas_id, gui::Canvas* canvas); + + // Register an overworld editor (for tile get/set callbacks) + void RegisterOverworldEditor(const std::string& canvas_id, + editor::OverworldEditor* editor); + + // RPC method implementations + absl::Status SetTile(const proto::SetTileRequest* request, + proto::SetTileResponse* response); + + absl::Status GetTile(const proto::GetTileRequest* request, + proto::GetTileResponse* response); + + absl::Status SetTiles(const proto::SetTilesRequest* request, + proto::SetTilesResponse* response); + + absl::Status SelectTile(const proto::SelectTileRequest* request, + proto::SelectTileResponse* response); + + absl::Status SelectTileRect(const proto::SelectTileRectRequest* request, + proto::SelectTileRectResponse* response); + + absl::Status GetSelection(const proto::GetSelectionRequest* request, + proto::GetSelectionResponse* response); + + absl::Status ClearSelection(const proto::ClearSelectionRequest* request, + proto::ClearSelectionResponse* response); + + absl::Status ScrollToTile(const proto::ScrollToTileRequest* request, + proto::ScrollToTileResponse* response); + + absl::Status CenterOn(const proto::CenterOnRequest* request, + proto::CenterOnResponse* response); + + absl::Status SetZoom(const proto::SetZoomRequest* request, + proto::SetZoomResponse* response); + + absl::Status GetZoom(const proto::GetZoomRequest* request, + proto::GetZoomResponse* response); + + absl::Status GetDimensions(const proto::GetDimensionsRequest* request, + proto::GetDimensionsResponse* response); + + absl::Status GetVisibleRegion(const proto::GetVisibleRegionRequest* request, + proto::GetVisibleRegionResponse* response); + + absl::Status IsTileVisible(const proto::IsTileVisibleRequest* request, + proto::IsTileVisibleResponse* response); + + private: + gui::Canvas* GetCanvas(const std::string& canvas_id); + editor::OverworldEditor* GetOverworldEditor(const std::string& canvas_id); + + // Canvas registry + std::unordered_map canvases_; + + // Editor registry (for tile callbacks) + std::unordered_map overworld_editors_; +}; + +} // namespace yaze + +#endif // YAZE_WITH_GRPC +#endif // YAZE_APP_CORE_SERVICE_CANVAS_AUTOMATION_SERVICE_H_ + diff --git a/src/protos/canvas_automation.proto b/src/protos/canvas_automation.proto new file mode 100644 index 00000000..08ca7533 --- /dev/null +++ b/src/protos/canvas_automation.proto @@ -0,0 +1,236 @@ +syntax = "proto3"; + +package yaze.proto; + +// ============================================================================ +// Canvas Automation Service +// ============================================================================ +// Provides remote control of canvas operations for: +// - AI agents via tool calls +// - ImGuiTestHarness automation +// - Remote GUI testing and validation +// - Batch scripting workflows +// - Coordinate conversion independent of zoom/scroll +// +// All operations use logical tile coordinates, not pixel coordinates. +// This ensures commands work consistently regardless of UI zoom level. + +service CanvasAutomation { + // Tile Operations + rpc SetTile(SetTileRequest) returns (SetTileResponse); + rpc GetTile(GetTileRequest) returns (GetTileResponse); + rpc SetTiles(SetTilesRequest) returns (SetTilesResponse); + + // Selection Operations + rpc SelectTile(SelectTileRequest) returns (SelectTileResponse); + rpc SelectTileRect(SelectTileRectRequest) returns (SelectTileRectResponse); + rpc GetSelection(GetSelectionRequest) returns (GetSelectionResponse); + rpc ClearSelection(ClearSelectionRequest) returns (ClearSelectionResponse); + + // View Operations + rpc ScrollToTile(ScrollToTileRequest) returns (ScrollToTileResponse); + rpc CenterOn(CenterOnRequest) returns (CenterOnResponse); + rpc SetZoom(SetZoomRequest) returns (SetZoomResponse); + rpc GetZoom(GetZoomRequest) returns (GetZoomResponse); + + // Query Operations + rpc GetDimensions(GetDimensionsRequest) returns (GetDimensionsResponse); + rpc GetVisibleRegion(GetVisibleRegionRequest) returns (GetVisibleRegionResponse); + rpc IsTileVisible(IsTileVisibleRequest) returns (IsTileVisibleResponse); +} + +// ============================================================================ +// Common Types +// ============================================================================ + +message TileCoord { + int32 x = 1; + int32 y = 2; +} + +message TileData { + int32 x = 1; + int32 y = 2; + int32 tile_id = 3; +} + +message Rect { + int32 x1 = 1; + int32 y1 = 2; + int32 x2 = 3; + int32 y2 = 4; +} + +message Dimensions { + int32 width_tiles = 1; + int32 height_tiles = 2; + int32 tile_size = 3; +} + +message VisibleRegion { + int32 min_x = 1; + int32 min_y = 2; + int32 max_x = 3; + int32 max_y = 4; +} + +// ============================================================================ +// Tile Operations +// ============================================================================ + +message SetTileRequest { + string canvas_id = 1; + int32 x = 2; + int32 y = 3; + int32 tile_id = 4; +} + +message SetTileResponse { + bool success = 1; + string error = 2; +} + +message GetTileRequest { + string canvas_id = 1; + int32 x = 2; + int32 y = 3; +} + +message GetTileResponse { + int32 tile_id = 1; + bool success = 2; + string error = 3; +} + +message SetTilesRequest { + string canvas_id = 1; + repeated TileData tiles = 2; +} + +message SetTilesResponse { + bool success = 1; + int32 tiles_painted = 2; + string error = 3; +} + +// ============================================================================ +// Selection Operations +// ============================================================================ + +message SelectTileRequest { + string canvas_id = 1; + int32 x = 2; + int32 y = 3; +} + +message SelectTileResponse { + bool success = 1; + string error = 2; +} + +message SelectTileRectRequest { + string canvas_id = 1; + Rect rect = 2; +} + +message SelectTileRectResponse { + bool success = 1; + int32 tiles_selected = 2; + string error = 3; +} + +message GetSelectionRequest { + string canvas_id = 1; +} + +message GetSelectionResponse { + bool has_selection = 1; + repeated TileCoord selected_tiles = 2; + TileCoord selection_start = 3; + TileCoord selection_end = 4; +} + +message ClearSelectionRequest { + string canvas_id = 1; +} + +message ClearSelectionResponse { + bool success = 1; +} + +// ============================================================================ +// View Operations +// ============================================================================ + +message ScrollToTileRequest { + string canvas_id = 1; + int32 x = 2; + int32 y = 3; + bool center = 4; +} + +message ScrollToTileResponse { + bool success = 1; + string error = 2; +} + +message CenterOnRequest { + string canvas_id = 1; + int32 x = 2; + int32 y = 3; +} + +message CenterOnResponse { + bool success = 1; + string error = 2; +} + +message SetZoomRequest { + string canvas_id = 1; + float zoom = 2; +} + +message SetZoomResponse { + bool success = 1; + float actual_zoom = 2; // May be clamped + string error = 3; +} + +message GetZoomRequest { + string canvas_id = 1; +} + +message GetZoomResponse { + float zoom = 1; +} + +// ============================================================================ +// Query Operations +// ============================================================================ + +message GetDimensionsRequest { + string canvas_id = 1; +} + +message GetDimensionsResponse { + Dimensions dimensions = 1; +} + +message GetVisibleRegionRequest { + string canvas_id = 1; +} + +message GetVisibleRegionResponse { + VisibleRegion region = 1; +} + +message IsTileVisibleRequest { + string canvas_id = 1; + int32 x = 2; + int32 y = 3; +} + +message IsTileVisibleResponse { + bool is_visible = 1; +} + diff --git a/src/app/core/proto/imgui_test_harness.proto b/src/protos/imgui_test_harness.proto similarity index 100% rename from src/app/core/proto/imgui_test_harness.proto rename to src/protos/imgui_test_harness.proto