epic: refactor SDL2_Renderer usage to IRenderer and queued texture rendering
- Updated the testing guide to clarify the testing framework's organization and execution methods, improving user understanding. - Refactored CMakeLists to include new platform-specific files, ensuring proper integration of the rendering backend. - Modified main application files to utilize the new IRenderer interface, enhancing flexibility in rendering operations. - Implemented deferred texture management in various components, allowing for more efficient graphics handling and improved performance. - Introduced new methods for texture creation and updates, streamlining the rendering process across the application. - Enhanced logging and error handling in the rendering pipeline to facilitate better debugging and diagnostics.
This commit is contained in:
scawful
334
test/integration/ai/ai_gui_controller_test.cc
Normal file
334
test/integration/ai/ai_gui_controller_test.cc
Normal file
@@ -0,0 +1,334 @@
|
||||
// Integration tests for AIGUIController
|
||||
// Tests the gRPC GUI automation with vision feedback
|
||||
|
||||
#include "cli/service/ai/ai_gui_controller.h"
|
||||
|
||||
#include <gmock/gmock.h>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "cli/service/ai/gemini_ai_service.h"
|
||||
#include "cli/service/gui/gui_automation_client.h"
|
||||
|
||||
namespace yaze {
|
||||
namespace cli {
|
||||
namespace ai {
|
||||
namespace {
|
||||
|
||||
using ::testing::_;
|
||||
using ::testing::Return;
|
||||
|
||||
// Mock GuiAutomationClient for testing without actual GUI
|
||||
class MockGuiAutomationClient : public GuiAutomationClient {
|
||||
public:
|
||||
MockGuiAutomationClient() : GuiAutomationClient("localhost:50052") {}
|
||||
|
||||
MOCK_METHOD(absl::Status, Connect, ());
|
||||
MOCK_METHOD(absl::StatusOr<AutomationResult>, Ping, (const std::string&));
|
||||
MOCK_METHOD(absl::StatusOr<AutomationResult>, Click,
|
||||
(const std::string&, ClickType));
|
||||
MOCK_METHOD(absl::StatusOr<AutomationResult>, Type,
|
||||
(const std::string&, const std::string&, bool));
|
||||
MOCK_METHOD(absl::StatusOr<AutomationResult>, Wait,
|
||||
(const std::string&, int, int));
|
||||
MOCK_METHOD(absl::StatusOr<AutomationResult>, Assert,
|
||||
(const std::string&));
|
||||
};
|
||||
|
||||
class AIGUIControllerTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
// Create mock services
|
||||
GeminiConfig config;
|
||||
config.api_key = "test_key";
|
||||
config.model = "gemini-2.5-flash";
|
||||
gemini_service_ = std::make_unique<GeminiAIService>(config);
|
||||
|
||||
gui_client_ = std::make_unique<MockGuiAutomationClient>();
|
||||
|
||||
controller_ = std::make_unique<AIGUIController>(
|
||||
gemini_service_.get(), gui_client_.get());
|
||||
|
||||
ControlLoopConfig loop_config;
|
||||
loop_config.max_iterations = 5;
|
||||
loop_config.enable_vision_verification = false; // Disable for unit tests
|
||||
loop_config.enable_iterative_refinement = false;
|
||||
controller_->Initialize(loop_config);
|
||||
}
|
||||
|
||||
std::unique_ptr<GeminiAIService> gemini_service_;
|
||||
std::unique_ptr<MockGuiAutomationClient> gui_client_;
|
||||
std::unique_ptr<AIGUIController> controller_;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Basic Action Execution Tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteClickAction_Success) {
|
||||
AIAction action(AIActionType::kClickButton);
|
||||
action.parameters["target"] = "button:Test";
|
||||
action.parameters["click_type"] = "left";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = true;
|
||||
result.message = "Click successful";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Click("button:Test", ClickType::kLeft))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
ASSERT_TRUE(status.ok()) << status.status().message();
|
||||
EXPECT_TRUE(status->action_successful);
|
||||
}
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteClickAction_Failure) {
|
||||
AIAction action(AIActionType::kClickButton);
|
||||
action.parameters["target"] = "button:NonExistent";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = false;
|
||||
result.message = "Button not found";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Click("button:NonExistent", ClickType::kLeft))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(status.status().message(),
|
||||
::testing::HasSubstr("Click action failed"));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Type Action Tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteTypeAction_Success) {
|
||||
AIAction action(AIActionType::kSelectTile); // Using SelectTile as a type action
|
||||
action.parameters["target"] = "input:TileID";
|
||||
action.parameters["text"] = "0x42";
|
||||
action.parameters["clear_first"] = "true";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = true;
|
||||
result.message = "Text entered";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Type("input:TileID", "0x42", true))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
ASSERT_TRUE(status.ok());
|
||||
EXPECT_TRUE(status->action_successful);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Wait Action Tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteWaitAction_Success) {
|
||||
AIAction action(AIActionType::kWait);
|
||||
action.parameters["condition"] = "window:OverworldEditor";
|
||||
action.parameters["timeout_ms"] = "2000";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = true;
|
||||
result.message = "Condition met";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Wait("window:OverworldEditor", 2000, 100))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
ASSERT_TRUE(status.ok());
|
||||
EXPECT_TRUE(status->action_successful);
|
||||
}
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteWaitAction_Timeout) {
|
||||
AIAction action(AIActionType::kWait);
|
||||
action.parameters["condition"] = "window:NonExistentWindow";
|
||||
action.parameters["timeout_ms"] = "100";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = false;
|
||||
result.message = "Timeout waiting for condition";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Wait("window:NonExistentWindow", 100, 100))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
EXPECT_FALSE(status.ok());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Verify/Assert Action Tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteVerifyAction_Success) {
|
||||
AIAction action(AIActionType::kVerifyTile);
|
||||
action.parameters["condition"] = "tile_placed";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = true;
|
||||
result.message = "Assertion passed";
|
||||
result.expected_value = "0x42";
|
||||
result.actual_value = "0x42";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Assert("tile_placed"))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
ASSERT_TRUE(status.ok());
|
||||
EXPECT_TRUE(status->action_successful);
|
||||
}
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteVerifyAction_Failure) {
|
||||
AIAction action(AIActionType::kVerifyTile);
|
||||
action.parameters["condition"] = "tile_placed";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = false;
|
||||
result.message = "Assertion failed";
|
||||
result.expected_value = "0x42";
|
||||
result.actual_value = "0x00";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Assert("tile_placed"))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(status.status().message(),
|
||||
::testing::HasSubstr("Assert action failed"));
|
||||
EXPECT_THAT(status.status().message(),
|
||||
::testing::HasSubstr("expected: 0x42"));
|
||||
EXPECT_THAT(status.status().message(),
|
||||
::testing::HasSubstr("actual: 0x00"));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Complex Tile Placement Action Tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecutePlaceTileAction_CompleteFlow) {
|
||||
AIAction action(AIActionType::kPlaceTile);
|
||||
action.parameters["map_id"] = "5";
|
||||
action.parameters["x"] = "10";
|
||||
action.parameters["y"] = "20";
|
||||
action.parameters["tile"] = "0x42";
|
||||
|
||||
AutomationResult result;
|
||||
result.success = true;
|
||||
|
||||
// Expect sequence: open menu, wait for window, set map ID, click position
|
||||
testing::InSequence seq;
|
||||
|
||||
EXPECT_CALL(*gui_client_, Click("menu:Overworld", ClickType::kLeft))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
EXPECT_CALL(*gui_client_, Wait("window:Overworld Editor", 2000, 100))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
EXPECT_CALL(*gui_client_, Type("input:Map ID", "5", true))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
EXPECT_CALL(*gui_client_, Click(::testing::_, ClickType::kLeft))
|
||||
.WillOnce(Return(result));
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
ASSERT_TRUE(status.ok()) << status.status().message();
|
||||
EXPECT_TRUE(status->action_successful);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Multiple Actions Execution Tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteActions_MultipleActionsSuccess) {
|
||||
std::vector<AIAction> actions;
|
||||
|
||||
AIAction action1(AIActionType::kClickButton);
|
||||
action1.parameters["target"] = "button:Overworld";
|
||||
actions.push_back(action1);
|
||||
|
||||
AIAction action2(AIActionType::kWait);
|
||||
action2.parameters["condition"] = "window:OverworldEditor";
|
||||
actions.push_back(action2);
|
||||
|
||||
AutomationResult success_result;
|
||||
success_result.success = true;
|
||||
|
||||
EXPECT_CALL(*gui_client_, Click("button:Overworld", ClickType::kLeft))
|
||||
.WillOnce(Return(success_result));
|
||||
|
||||
EXPECT_CALL(*gui_client_, Wait("window:OverworldEditor", 5000, 100))
|
||||
.WillOnce(Return(success_result));
|
||||
|
||||
auto result = controller_->ExecuteActions(actions);
|
||||
|
||||
ASSERT_TRUE(result.ok()) << result.status().message();
|
||||
EXPECT_TRUE(result->success);
|
||||
EXPECT_EQ(result->actions_executed.size(), 2);
|
||||
}
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteActions_StopsOnFirstFailure) {
|
||||
std::vector<AIAction> actions;
|
||||
|
||||
AIAction action1(AIActionType::kClickButton);
|
||||
action1.parameters["target"] = "button:Test";
|
||||
actions.push_back(action1);
|
||||
|
||||
AIAction action2(AIActionType::kClickButton);
|
||||
action2.parameters["target"] = "button:NeverReached";
|
||||
actions.push_back(action2);
|
||||
|
||||
AutomationResult failure_result;
|
||||
failure_result.success = false;
|
||||
failure_result.message = "First action failed";
|
||||
|
||||
EXPECT_CALL(*gui_client_, Click("button:Test", ClickType::kLeft))
|
||||
.WillOnce(Return(failure_result));
|
||||
|
||||
// Second action should never be called
|
||||
EXPECT_CALL(*gui_client_, Click("button:NeverReached", _))
|
||||
.Times(0);
|
||||
|
||||
auto result = controller_->ExecuteActions(actions);
|
||||
|
||||
EXPECT_FALSE(result.ok());
|
||||
EXPECT_EQ(result->actions_executed.size(), 1);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Handling Tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecuteAction_InvalidActionType) {
|
||||
AIAction action(AIActionType::kInvalidAction);
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(status.status().message(),
|
||||
::testing::HasSubstr("Action type not implemented"));
|
||||
}
|
||||
|
||||
TEST_F(AIGUIControllerTest, ExecutePlaceTileAction_MissingParameters) {
|
||||
AIAction action(AIActionType::kPlaceTile);
|
||||
// Missing required parameters
|
||||
|
||||
auto status = controller_->ExecuteSingleAction(action, false);
|
||||
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(status.status().message(),
|
||||
::testing::HasSubstr("requires map_id, x, y, and tile"));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace ai
|
||||
} // namespace cli
|
||||
} // namespace yaze
|
||||
211
test/integration/ai/test_ai_tile_placement.cc
Normal file
211
test/integration/ai/test_ai_tile_placement.cc
Normal file
@@ -0,0 +1,211 @@
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "cli/service/ai/ai_action_parser.h"
|
||||
#include "cli/service/ai/vision_action_refiner.h"
|
||||
#include "cli/service/ai/ai_gui_controller.h"
|
||||
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
#include "cli/service/gui/gui_automation_client.h"
|
||||
#include "cli/service/ai/gemini_ai_service.h"
|
||||
#endif
|
||||
|
||||
namespace yaze {
|
||||
namespace test {
|
||||
|
||||
/**
|
||||
* @brief Integration tests for AI-controlled tile placement
|
||||
*
|
||||
* These tests verify the complete pipeline:
|
||||
* 1. Parse natural language commands
|
||||
* 2. Execute actions via gRPC
|
||||
* 3. Verify success with vision analysis
|
||||
* 4. Refine and retry on failure
|
||||
*/
|
||||
class AITilePlacementTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
// These tests require YAZE GUI to be running with gRPC test harness
|
||||
// Skip if not available
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(AITilePlacementTest, ParsePlaceTileCommand) {
|
||||
using namespace cli::ai;
|
||||
|
||||
// Test basic tile placement command
|
||||
auto result = AIActionParser::ParseCommand(
|
||||
"Place tile 0x42 at overworld position (5, 7)");
|
||||
|
||||
ASSERT_TRUE(result.ok()) << result.status().message();
|
||||
EXPECT_EQ(result->size(), 3); // Select, Place, Save
|
||||
|
||||
// Check first action (Select)
|
||||
EXPECT_EQ(result->at(0).type, AIActionType::kSelectTile);
|
||||
EXPECT_EQ(result->at(0).parameters.at("tile_id"), "66"); // 0x42 = 66
|
||||
|
||||
// Check second action (Place)
|
||||
EXPECT_EQ(result->at(1).type, AIActionType::kPlaceTile);
|
||||
EXPECT_EQ(result->at(1).parameters.at("x"), "5");
|
||||
EXPECT_EQ(result->at(1).parameters.at("y"), "7");
|
||||
EXPECT_EQ(result->at(1).parameters.at("map_id"), "0");
|
||||
|
||||
// Check third action (Save)
|
||||
EXPECT_EQ(result->at(2).type, AIActionType::kSaveTile);
|
||||
}
|
||||
|
||||
TEST_F(AITilePlacementTest, ParseSelectTileCommand) {
|
||||
using namespace cli::ai;
|
||||
|
||||
auto result = AIActionParser::ParseCommand("Select tile 100");
|
||||
|
||||
ASSERT_TRUE(result.ok());
|
||||
EXPECT_EQ(result->size(), 1);
|
||||
EXPECT_EQ(result->at(0).type, AIActionType::kSelectTile);
|
||||
EXPECT_EQ(result->at(0).parameters.at("tile_id"), "100");
|
||||
}
|
||||
|
||||
TEST_F(AITilePlacementTest, ParseOpenEditorCommand) {
|
||||
using namespace cli::ai;
|
||||
|
||||
auto result = AIActionParser::ParseCommand("Open the overworld editor");
|
||||
|
||||
ASSERT_TRUE(result.ok());
|
||||
EXPECT_EQ(result->size(), 1);
|
||||
EXPECT_EQ(result->at(0).type, AIActionType::kOpenEditor);
|
||||
EXPECT_EQ(result->at(0).parameters.at("editor"), "overworld");
|
||||
}
|
||||
|
||||
TEST_F(AITilePlacementTest, ActionToStringRoundtrip) {
|
||||
using namespace cli::ai;
|
||||
|
||||
AIAction action(AIActionType::kPlaceTile, {
|
||||
{"x", "5"},
|
||||
{"y", "7"},
|
||||
{"tile_id", "42"}
|
||||
});
|
||||
|
||||
std::string str = AIActionParser::ActionToString(action);
|
||||
EXPECT_FALSE(str.empty());
|
||||
EXPECT_TRUE(str.find("5") != std::string::npos);
|
||||
EXPECT_TRUE(str.find("7") != std::string::npos);
|
||||
}
|
||||
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
|
||||
TEST_F(AITilePlacementTest, DISABLED_VisionAnalysisBasic) {
|
||||
// This test requires Gemini API key
|
||||
const char* api_key = std::getenv("GEMINI_API_KEY");
|
||||
if (!api_key || std::string(api_key).empty()) {
|
||||
GTEST_SKIP() << "GEMINI_API_KEY not set";
|
||||
}
|
||||
|
||||
cli::GeminiConfig config;
|
||||
config.api_key = api_key;
|
||||
config.model = "gemini-2.5-flash";
|
||||
|
||||
cli::GeminiAIService gemini_service(config);
|
||||
cli::ai::VisionActionRefiner refiner(&gemini_service);
|
||||
|
||||
// Would need actual screenshots for real test
|
||||
// This is a structure test
|
||||
EXPECT_TRUE(true);
|
||||
}
|
||||
|
||||
TEST_F(AITilePlacementTest, DISABLED_FullAIControlLoop) {
|
||||
// This test requires:
|
||||
// 1. YAZE GUI running with gRPC test harness
|
||||
// 2. Gemini API key for vision
|
||||
// 3. Test ROM loaded
|
||||
|
||||
const char* api_key = std::getenv("GEMINI_API_KEY");
|
||||
if (!api_key || std::string(api_key).empty()) {
|
||||
GTEST_SKIP() << "GEMINI_API_KEY not set";
|
||||
}
|
||||
|
||||
// Initialize services
|
||||
cli::GeminiConfig gemini_config;
|
||||
gemini_config.api_key = api_key;
|
||||
cli::GeminiAIService gemini_service(gemini_config);
|
||||
|
||||
cli::GuiAutomationClient gui_client("localhost:50051");
|
||||
auto connect_status = gui_client.Connect();
|
||||
if (!connect_status.ok()) {
|
||||
GTEST_SKIP() << "GUI test harness not available: "
|
||||
<< connect_status.message();
|
||||
}
|
||||
|
||||
// Create AI controller
|
||||
cli::ai::AIGUIController controller(&gemini_service, &gui_client);
|
||||
cli::ai::ControlLoopConfig config;
|
||||
config.max_iterations = 5;
|
||||
config.enable_vision_verification = true;
|
||||
controller.Initialize(config);
|
||||
|
||||
// Execute command
|
||||
auto result = controller.ExecuteCommand(
|
||||
"Place tile 0x42 at overworld position (5, 7)");
|
||||
|
||||
if (result.ok()) {
|
||||
EXPECT_TRUE(result->success);
|
||||
EXPECT_GT(result->iterations_performed, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#endif // YAZE_WITH_GRPC
|
||||
|
||||
TEST_F(AITilePlacementTest, ActionRefinement) {
|
||||
using namespace cli::ai;
|
||||
|
||||
// Test refinement logic with a failed action
|
||||
VisionAnalysisResult analysis;
|
||||
analysis.action_successful = false;
|
||||
analysis.error_message = "Element not found";
|
||||
|
||||
AIAction original_action(AIActionType::kClickButton, {
|
||||
{"button", "save"}
|
||||
});
|
||||
|
||||
// Would need VisionActionRefiner for real test
|
||||
// This verifies the structure compiles
|
||||
EXPECT_TRUE(true);
|
||||
}
|
||||
|
||||
TEST_F(AITilePlacementTest, MultipleCommandsParsing) {
|
||||
using namespace cli::ai;
|
||||
|
||||
// Test that we can parse multiple commands in sequence
|
||||
std::vector<std::string> commands = {
|
||||
"Open overworld editor",
|
||||
"Select tile 0x42",
|
||||
"Place tile at position (5, 7)",
|
||||
"Save changes"
|
||||
};
|
||||
|
||||
for (const auto& cmd : commands) {
|
||||
auto result = AIActionParser::ParseCommand(cmd);
|
||||
// At least some should parse successfully
|
||||
if (result.ok()) {
|
||||
EXPECT_FALSE(result->empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(AITilePlacementTest, HexAndDecimalParsing) {
|
||||
using namespace cli::ai;
|
||||
|
||||
// Test hex notation
|
||||
auto hex_result = AIActionParser::ParseCommand("Select tile 0xFF");
|
||||
if (hex_result.ok() && !hex_result->empty()) {
|
||||
EXPECT_EQ(hex_result->at(0).parameters.at("tile_id"), "255");
|
||||
}
|
||||
|
||||
// Test decimal notation
|
||||
auto dec_result = AIActionParser::ParseCommand("Select tile 255");
|
||||
if (dec_result.ok() && !dec_result->empty()) {
|
||||
EXPECT_EQ(dec_result->at(0).parameters.at("tile_id"), "255");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace yaze
|
||||
243
test/integration/ai/test_gemini_vision.cc
Normal file
243
test/integration/ai/test_gemini_vision.cc
Normal file
@@ -0,0 +1,243 @@
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "cli/service/ai/gemini_ai_service.h"
|
||||
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
#include "app/core/service/screenshot_utils.h"
|
||||
#endif
|
||||
|
||||
namespace yaze {
|
||||
namespace test {
|
||||
|
||||
class GeminiVisionTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
// Check if GEMINI_API_KEY is set
|
||||
const char* api_key = std::getenv("GEMINI_API_KEY");
|
||||
if (!api_key || std::string(api_key).empty()) {
|
||||
GTEST_SKIP() << "GEMINI_API_KEY not set. Skipping multimodal tests.";
|
||||
}
|
||||
|
||||
api_key_ = api_key;
|
||||
|
||||
// Create test data directory
|
||||
test_dir_ = std::filesystem::temp_directory_path() / "yaze_multimodal_test";
|
||||
std::filesystem::create_directories(test_dir_);
|
||||
}
|
||||
|
||||
void TearDown() override {
|
||||
// Clean up test directory
|
||||
if (std::filesystem::exists(test_dir_)) {
|
||||
std::filesystem::remove_all(test_dir_);
|
||||
}
|
||||
}
|
||||
|
||||
// Helper: Create a simple test image (16x16 PNG)
|
||||
std::filesystem::path CreateTestImage() {
|
||||
auto image_path = test_dir_ / "test_image.png";
|
||||
|
||||
// Create a minimal PNG file (16x16 red square)
|
||||
// PNG signature + IHDR + IDAT + IEND
|
||||
const unsigned char png_data[] = {
|
||||
// PNG signature
|
||||
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A,
|
||||
// IHDR chunk
|
||||
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
|
||||
0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10,
|
||||
0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x91, 0x68,
|
||||
0x36,
|
||||
// IDAT chunk (minimal data)
|
||||
0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, 0x54,
|
||||
0x08, 0x99, 0x63, 0xF8, 0xCF, 0xC0, 0x00, 0x00,
|
||||
0x03, 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D, 0xB4,
|
||||
// IEND chunk
|
||||
0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44,
|
||||
0xAE, 0x42, 0x60, 0x82
|
||||
};
|
||||
|
||||
std::ofstream file(image_path, std::ios::binary);
|
||||
file.write(reinterpret_cast<const char*>(png_data), sizeof(png_data));
|
||||
file.close();
|
||||
|
||||
return image_path;
|
||||
}
|
||||
|
||||
std::string api_key_;
|
||||
std::filesystem::path test_dir_;
|
||||
};
|
||||
|
||||
TEST_F(GeminiVisionTest, BasicImageAnalysis) {
|
||||
cli::GeminiConfig config;
|
||||
config.api_key = api_key_;
|
||||
config.model = "gemini-2.5-flash"; // Vision-capable model
|
||||
config.verbose = false;
|
||||
|
||||
cli::GeminiAIService service(config);
|
||||
|
||||
// Create test image
|
||||
auto image_path = CreateTestImage();
|
||||
ASSERT_TRUE(std::filesystem::exists(image_path));
|
||||
|
||||
// Send multimodal request
|
||||
auto response = service.GenerateMultimodalResponse(
|
||||
image_path.string(),
|
||||
"Describe this image in one sentence."
|
||||
);
|
||||
|
||||
ASSERT_TRUE(response.ok()) << response.status().message();
|
||||
EXPECT_FALSE(response->text_response.empty());
|
||||
|
||||
std::cout << "Vision API response: " << response->text_response << std::endl;
|
||||
}
|
||||
|
||||
TEST_F(GeminiVisionTest, ImageWithSpecificPrompt) {
|
||||
cli::GeminiConfig config;
|
||||
config.api_key = api_key_;
|
||||
config.model = "gemini-2.5-flash";
|
||||
config.verbose = false;
|
||||
|
||||
cli::GeminiAIService service(config);
|
||||
|
||||
auto image_path = CreateTestImage();
|
||||
|
||||
// Ask specific question about the image
|
||||
auto response = service.GenerateMultimodalResponse(
|
||||
image_path.string(),
|
||||
"What color is the dominant color in this image? Answer with just the color name."
|
||||
);
|
||||
|
||||
ASSERT_TRUE(response.ok()) << response.status().message();
|
||||
EXPECT_FALSE(response->text_response.empty());
|
||||
|
||||
// Response should mention "red" since we created a red square
|
||||
std::string response_lower = response->text_response;
|
||||
std::transform(response_lower.begin(), response_lower.end(),
|
||||
response_lower.begin(), ::tolower);
|
||||
EXPECT_TRUE(response_lower.find("red") != std::string::npos ||
|
||||
response_lower.find("pink") != std::string::npos)
|
||||
<< "Expected color 'red' or 'pink' in response: " << response->text_response;
|
||||
}
|
||||
|
||||
TEST_F(GeminiVisionTest, InvalidImagePath) {
|
||||
cli::GeminiConfig config;
|
||||
config.api_key = api_key_;
|
||||
config.model = "gemini-2.5-flash";
|
||||
|
||||
cli::GeminiAIService service(config);
|
||||
|
||||
// Try with non-existent image
|
||||
auto response = service.GenerateMultimodalResponse(
|
||||
"/nonexistent/image.png",
|
||||
"Describe this image."
|
||||
);
|
||||
|
||||
EXPECT_FALSE(response.ok());
|
||||
EXPECT_TRUE(absl::IsNotFound(response.status()) ||
|
||||
absl::IsInternal(response.status()));
|
||||
}
|
||||
|
||||
#ifdef YAZE_WITH_GRPC
|
||||
// Integration test with screenshot capture
|
||||
TEST_F(GeminiVisionTest, ScreenshotCaptureIntegration) {
|
||||
// Note: This test requires a running YAZE instance with gRPC test harness
|
||||
// Skip if we can't connect
|
||||
|
||||
cli::GeminiConfig config;
|
||||
config.api_key = api_key_;
|
||||
config.model = "gemini-2.5-flash";
|
||||
config.verbose = false;
|
||||
|
||||
cli::GeminiAIService service(config);
|
||||
|
||||
// Attempt to capture a screenshot
|
||||
auto screenshot_result = yaze::test::CaptureHarnessScreenshot(
|
||||
(test_dir_ / "screenshot.png").string());
|
||||
|
||||
if (!screenshot_result.ok()) {
|
||||
GTEST_SKIP() << "Screenshot capture failed (YAZE may not be running): "
|
||||
<< screenshot_result.status().message();
|
||||
}
|
||||
|
||||
// Analyze the captured screenshot
|
||||
auto response = service.GenerateMultimodalResponse(
|
||||
screenshot_result->file_path,
|
||||
"What UI elements are visible in this screenshot? List them."
|
||||
);
|
||||
|
||||
ASSERT_TRUE(response.ok()) << response.status().message();
|
||||
EXPECT_FALSE(response->text_response.empty());
|
||||
|
||||
std::cout << "Screenshot analysis: " << response->text_response << std::endl;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Performance test
|
||||
TEST_F(GeminiVisionTest, MultipleRequestsSequential) {
|
||||
cli::GeminiConfig config;
|
||||
config.api_key = api_key_;
|
||||
config.model = "gemini-2.5-flash";
|
||||
config.verbose = false;
|
||||
|
||||
cli::GeminiAIService service(config);
|
||||
|
||||
auto image_path = CreateTestImage();
|
||||
|
||||
// Make 3 sequential requests
|
||||
const int num_requests = 3;
|
||||
for (int i = 0; i < num_requests; ++i) {
|
||||
auto response = service.GenerateMultimodalResponse(
|
||||
image_path.string(),
|
||||
absl::StrCat("Request ", i + 1, ": Describe this image briefly.")
|
||||
);
|
||||
|
||||
ASSERT_TRUE(response.ok()) << "Request " << i + 1 << " failed: "
|
||||
<< response.status().message();
|
||||
EXPECT_FALSE(response->text_response.empty());
|
||||
}
|
||||
}
|
||||
|
||||
// Rate limiting test (should handle gracefully)
|
||||
TEST_F(GeminiVisionTest, RateLimitHandling) {
|
||||
cli::GeminiConfig config;
|
||||
config.api_key = api_key_;
|
||||
config.model = "gemini-2.5-flash";
|
||||
config.verbose = false;
|
||||
|
||||
cli::GeminiAIService service(config);
|
||||
|
||||
auto image_path = CreateTestImage();
|
||||
|
||||
// Make many rapid requests (may hit rate limit)
|
||||
int successful = 0;
|
||||
int rate_limited = 0;
|
||||
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
auto response = service.GenerateMultimodalResponse(
|
||||
image_path.string(),
|
||||
"Describe this image."
|
||||
);
|
||||
|
||||
if (response.ok()) {
|
||||
successful++;
|
||||
} else if (absl::IsResourceExhausted(response.status()) ||
|
||||
response.status().message().find("429") != std::string::npos) {
|
||||
rate_limited++;
|
||||
}
|
||||
}
|
||||
|
||||
// At least some requests should succeed
|
||||
EXPECT_GT(successful, 0) << "No successful requests out of 10";
|
||||
|
||||
// If we hit rate limits, that's expected behavior (not a failure)
|
||||
if (rate_limited > 0) {
|
||||
std::cout << "Note: Hit rate limit on " << rate_limited << " out of 10 requests (expected)" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace yaze
|
||||
|
||||
// Note: main() is provided by yaze_test.cc for the unified test runner
|
||||
Reference in New Issue
Block a user