diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4328f9ef..5c316d60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,13 +184,13 @@ jobs: # Test (stable core functionality only for CI) - name: Run Core Tests working-directory: ${{ github.workspace }}/build - run: ctest --build-config ${{ env.BUILD_TYPE }} --output-on-failure -j1 -R "AsarWrapperTest|SnesTileTest|CompressionTest|SnesPaletteTest|HexTest" + run: ctest --build-config ${{ env.BUILD_TYPE }} --output-on-failure -j1 -R "AsarWrapperTest|SnesTileTest|CompressionTest|SnesPaletteTest|HexTest|MessageTest" - # Run experimental tests separately (allowed to fail for information only) - - name: Run Additional Tests (Informational) + # Run additional unit tests (allowed to fail for information only) + - name: Run Additional Unit Tests (Informational) working-directory: ${{ github.workspace }}/build continue-on-error: true - run: ctest --build-config ${{ env.BUILD_TYPE }} --output-on-failure --parallel -E "AsarWrapperTest|SnesTileTest|CompressionTest|SnesPaletteTest|HexTest|CpuTest|Spc700Test|ApuTest|MessageTest|.*IntegrationTest" + run: ctest --build-config ${{ env.BUILD_TYPE }} --output-on-failure --parallel -R ".*Test" -E ".*RomTest.*|.*E2E.*|.*ZSCustomOverworld.*|.*IntegrationTest.*|CpuTest|Spc700Test|ApuTest" # Package (only on successful builds) - name: Package artifacts diff --git a/docs/02-build-instructions.md b/docs/02-build-instructions.md index 52dc3de8..1e4afd8a 100644 --- a/docs/02-build-instructions.md +++ b/docs/02-build-instructions.md @@ -194,6 +194,125 @@ cmake --preset release # macOS cmake -B build -DCMAKE_BUILD_TYPE=Release # All platforms ``` +## Testing System + +YAZE includes a comprehensive testing system with different test categories designed for various use cases: + +### Test Categories + +#### Unit Tests +- **Core functionality**: AsarWrapper, ROM operations, SNES tiles, palettes +- **Graphics**: Compression, tile unpacking, color conversion +- **Zelda3 components**: Message system, overworld, object parsing +- **Location**: `test/unit/` + +#### Integration Tests +- **ASAR integration**: Assembly compilation and ROM patching +- **Editor integration**: Tile16 editor, dungeon editor +- **Location**: `test/integration/` + +#### End-to-End (E2E) Tests +- **ROM-dependent tests**: Load, edit, save, reload, verify integrity +- **ZSCustomOverworld upgrade tests**: Vanilla → v2 → v3 upgrade paths +- **Location**: `test/e2e/` + +### Running Tests + +#### Local Development +```bash +# Run all tests +./build/test/yaze_test + +# Run specific test categories +./build/test/yaze_test --unit +./build/test/yaze_test --integration +./build/test/yaze_test --e2e --rom-path zelda3.sfc + +# Run with verbose output +./build/test/yaze_test --verbose + +# Get help +./build/test/yaze_test --help +``` + +#### CI/CD Environment +The CI builds use a simplified test executable (`yaze_test_ci.cc`) that: +- Excludes ROM-dependent tests (no ROM files available) +- Excludes E2E tests (require actual ROM files) +- Focuses on unit tests and core functionality +- Uses minimal configuration for reliability + +#### ROM-Dependent Tests +These tests require actual ROM files and are only available in local development: + +```bash +# E2E ROM tests (requires zelda3.sfc) +./build/test/yaze_test --e2e --rom-path zelda3.sfc + +# ZSCustomOverworld upgrade tests +./build/test/yaze_test --zscustomoverworld --rom-path zelda3.sfc +``` + +**Note**: ROM-dependent tests are automatically skipped in CI builds and minimal builds. + +### Test Organization + +``` +test/ +├── unit/ # Unit tests (CI-safe) +│ ├── core/ # Core functionality tests +│ ├── gfx/ # Graphics tests +│ └── zelda3/ # Zelda3-specific tests +├── integration/ # Integration tests (CI-safe) +├── e2e/ # End-to-end tests (ROM-dependent) +│ ├── rom_dependent/ # ROM load/save/edit tests +│ └── zscustomoverworld/ # Upgrade path tests +├── mocks/ # Mock objects for testing +├── assets/ # Test assets and ROMs +└── deprecated/ # Outdated tests (moved from emu/) +``` + +### Test Executables + +#### Development Build (`yaze_test.cc`) +- Full argument parsing for AI agents +- SDL initialization for graphics tests +- Support for all test categories +- ROM path configuration +- Verbose output options + +#### CI Build (`yaze_test_ci.cc`) +- Simplified main function +- No SDL initialization +- Automatic exclusion of ROM-dependent tests +- Minimal configuration for reliability +- Used when `YAZE_MINIMAL_BUILD=ON` + +### Test Configuration + +#### CMake Options +```bash +# Enable/disable test categories +-DYAZE_ENABLE_ROM_TESTS=ON # Enable ROM-dependent tests +-DYAZE_ENABLE_UI_TESTS=ON # Enable ImGui Test Engine +-DYAZE_ENABLE_EXPERIMENTAL_TESTS=ON # Enable experimental tests +-DYAZE_MINIMAL_BUILD=ON # Use CI test executable +``` + +#### Test Filters +The test system supports Google Test filters for selective execution: + +```bash +# Run only core tests +./build/test/yaze_test --gtest_filter="*Core*" + +# Exclude ROM tests +./build/test/yaze_test --gtest_filter="-*RomTest*" + +# Run specific test suite +./build/test/yaze_test --gtest_filter="AsarWrapperTest.*" +``` + ## IDE Integration ### Visual Studio (Windows) @@ -284,6 +403,7 @@ The project includes several PowerShell and Batch scripts to streamline Windows | CLI Tools | ✅ | ✅ | ❌ | | Test Suite | ✅ | ❌ | ✅ (limited) | | UI Testing | ✅ | ❌ | ❌ | +| ROM Tests | ✅ | ❌ | ❌ | ## CMake Compatibility @@ -312,11 +432,27 @@ cmake -B build \ ### GitHub Actions Workflows -The project includes three release workflows with different levels of complexity: +The project includes comprehensive CI/CD workflows: -- **`release-simplified.yml`**: Fast, basic release builds -- **`release.yml`**: Standard release builds with fallback mechanisms -- **`release-complex.yml`**: Comprehensive release builds with multiple fallback strategies +- **`ci.yml`**: Multi-platform CI with test execution +- **`release.yml`**: Automated release builds with packaging + +### Test Execution in CI + +**CI Test Strategy:** +- **Core Tests**: Always run (AsarWrapper, SnesTile, Compression, SnesPalette, Hex, Message) +- **Unit Tests**: Run with `continue-on-error=true` for information +- **ROM Tests**: Automatically excluded (no ROM files available) +- **E2E Tests**: Automatically excluded (require ROM files) + +**Test Filters in CI:** +```bash +# Core tests (must pass) +ctest -R "AsarWrapperTest|SnesTileTest|CompressionTest|SnesPaletteTest|HexTest|MessageTest" + +# Additional unit tests (informational) +ctest -R ".*Test" -E ".*RomTest.*|.*E2E.*|.*ZSCustomOverworld.*|.*IntegrationTest.*" +``` ### vcpkg Fallback Mechanisms @@ -414,6 +550,32 @@ python scripts/generate-vs-projects.py .\scripts\setup-windows-dev.ps1 ``` +### Test Issues + +**Test Discovery Failures:** +```bash +# Use CI test executable for minimal builds +cmake -B build -DYAZE_MINIMAL_BUILD=ON +cmake --build build + +# Check test executable +./build/test/yaze_test --help +``` + +**ROM Test Failures:** +```bash +# Ensure ROM file exists +ls -la zelda3.sfc + +# Run with explicit ROM path +./build/test/yaze_test --e2e --rom-path zelda3.sfc +``` + +**SDL Initialization Errors:** +- These are expected in CI builds +- Use minimal build configuration for CI compatibility +- For local development, ensure SDL2 is properly installed + ### Architecture Errors (macOS) ```bash # Clean and use ARM64-only preset @@ -458,3 +620,8 @@ cmake --build build - Check if vcpkg download failed (network issues) - The workflows automatically fall back to minimal builds - For persistent issues, check the workflow logs for specific error messages + +**Test Executable Issues:** +- Ensure the correct test executable is being used (CI vs development) +- Check that test filters are properly configured +- Verify that ROM-dependent tests are excluded in CI builds \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3ed8acac..fdd45210 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -12,53 +12,106 @@ foreach (file endforeach() # Main test executable with enhanced argument handling for AI agents -add_executable( - yaze_test - yaze_test.cc - test_editor.cc - test_editor.h - testing.h - test_utils.h +# Use CI version for minimal builds, full version for development +if(YAZE_MINIMAL_BUILD) + # CI/Minimal build: use simplified test executable + add_executable( + yaze_test + yaze_test_ci.cc + test_editor.cc + test_editor.h + testing.h + test_utils.h - # Unit Tests - unit/core/asar_wrapper_test.cc - unit/core/hex_test.cc - unit/rom/rom_test.cc - unit/gfx/snes_tile_test.cc - unit/gfx/compression_test.cc - unit/gfx/snes_palette_test.cc - unit/zelda3/message_test.cc - unit/zelda3/overworld_test.cc - unit/zelda3/object_parser_test.cc - unit/zelda3/object_parser_structs_test.cc - unit/zelda3/sprite_builder_test.cc - unit/zelda3/sprite_position_test.cc - unit/zelda3/test_dungeon_objects.cc - unit/zelda3/dungeon_component_unit_test.cc - - # Integration Tests - integration/asar_integration_test.cc - integration/asar_rom_test.cc - integration/dungeon_editor_test.cc - integration/dungeon_editor_test.h - integration/editor/tile16_editor_test.cc - integration/editor/editor_integration_test.cc - integration/editor/editor_integration_test.h - - # E2E Tests - e2e/rom_dependent/e2e_rom_test.cc - e2e/zscustomoverworld/zscustomoverworld_upgrade_test.cc - - # Legacy Integration Tests (to be migrated) - unit/zelda3/comprehensive_integration_test.cc - unit/zelda3/overworld_integration_test.cc - unit/zelda3/dungeon_integration_test.cc - unit/zelda3/dungeon_editor_system_integration_test.cc - unit/zelda3/dungeon_object_renderer_integration_test.cc - unit/zelda3/dungeon_object_renderer_mock_test.cc - unit/zelda3/dungeon_object_rendering_tests.cc - unit/zelda3/dungeon_room_test.cc -) + # Unit Tests + unit/core/asar_wrapper_test.cc + unit/core/hex_test.cc + unit/rom/rom_test.cc + unit/gfx/snes_tile_test.cc + unit/gfx/compression_test.cc + unit/gfx/snes_palette_test.cc + unit/zelda3/message_test.cc + unit/zelda3/overworld_test.cc + unit/zelda3/object_parser_test.cc + unit/zelda3/object_parser_structs_test.cc + unit/zelda3/sprite_builder_test.cc + unit/zelda3/sprite_position_test.cc + unit/zelda3/test_dungeon_objects.cc + unit/zelda3/dungeon_component_unit_test.cc + + # Integration Tests + integration/asar_integration_test.cc + integration/asar_rom_test.cc + integration/dungeon_editor_test.cc + integration/dungeon_editor_test.h + integration/editor/tile16_editor_test.cc + integration/editor/editor_integration_test.cc + integration/editor/editor_integration_test.h + + # E2E Tests (excluded in CI builds) + e2e/rom_dependent/e2e_rom_test.cc + e2e/zscustomoverworld/zscustomoverworld_upgrade_test.cc + + # Legacy Integration Tests (to be migrated) + unit/zelda3/comprehensive_integration_test.cc + unit/zelda3/overworld_integration_test.cc + unit/zelda3/dungeon_integration_test.cc + unit/zelda3/dungeon_editor_system_integration_test.cc + unit/zelda3/dungeon_object_renderer_integration_test.cc + unit/zelda3/dungeon_object_renderer_mock_test.cc + unit/zelda3/dungeon_object_rendering_tests.cc + unit/zelda3/dungeon_room_test.cc + ) +else() + # Development build: use full-featured test executable + add_executable( + yaze_test + yaze_test.cc + test_editor.cc + test_editor.h + testing.h + test_utils.h + + # Unit Tests + unit/core/asar_wrapper_test.cc + unit/core/hex_test.cc + unit/rom/rom_test.cc + unit/gfx/snes_tile_test.cc + unit/gfx/compression_test.cc + unit/gfx/snes_palette_test.cc + unit/zelda3/message_test.cc + unit/zelda3/overworld_test.cc + unit/zelda3/object_parser_test.cc + unit/zelda3/object_parser_structs_test.cc + unit/zelda3/sprite_builder_test.cc + unit/zelda3/sprite_position_test.cc + unit/zelda3/test_dungeon_objects.cc + unit/zelda3/dungeon_component_unit_test.cc + + # Integration Tests + integration/asar_integration_test.cc + integration/asar_rom_test.cc + integration/dungeon_editor_test.cc + integration/dungeon_editor_test.h + integration/editor/tile16_editor_test.cc + integration/editor/editor_integration_test.cc + integration/editor/editor_integration_test.h + + # E2E Tests (included in development builds) + e2e/rom_dependent/e2e_rom_test.cc + e2e/zscustomoverworld/zscustomoverworld_upgrade_test.cc + + # Legacy Integration Tests (to be migrated) + unit/zelda3/comprehensive_integration_test.cc + unit/zelda3/overworld_integration_test.cc + unit/zelda3/dungeon_integration_test.cc + unit/zelda3/dungeon_editor_system_integration_test.cc + unit/zelda3/dungeon_object_renderer_integration_test.cc + unit/zelda3/dungeon_object_renderer_mock_test.cc + unit/zelda3/dungeon_object_rendering_tests.cc + unit/zelda3/dungeon_room_test.cc + ) +endif() # Add vanilla value extraction utility (only for local development with ROM access) if(NOT YAZE_MINIMAL_BUILD AND YAZE_ENABLE_ROM_TESTS) diff --git a/test/yaze_test_ci.cc b/test/yaze_test_ci.cc new file mode 100644 index 00000000..18de515b --- /dev/null +++ b/test/yaze_test_ci.cc @@ -0,0 +1,46 @@ +// Simplified test executable for CI/CD builds +// This version removes complex argument parsing and SDL initialization +// to ensure reliable test discovery and execution in automated environments + +#include +#include + +#include "absl/debugging/failure_signal_handler.h" +#include "absl/debugging/symbolize.h" + +int main(int argc, char* argv[]) { + // Initialize symbolizer for better error reporting + absl::InitializeSymbolizer(argv[0]); + + // Configure failure signal handler to be less aggressive for CI + absl::FailureSignalHandlerOptions options; + options.symbolize_stacktrace = true; + options.use_alternate_stack = false; + options.alarm_on_failure_secs = false; + options.call_previous_handler = true; + options.writerfn = nullptr; + absl::InstallFailureSignalHandler(options); + + // Initialize Google Test with minimal configuration + ::testing::InitGoogleTest(&argc, argv); + + // Set up basic test environment + ::testing::FLAGS_gtest_color = "yes"; + ::testing::FLAGS_gtest_print_time = true; + + // For CI builds, skip ROM-dependent tests by default + // These tests require actual ROM files which aren't available in CI + std::string filter = ::testing::GTEST_FLAG(filter); + if (filter.empty()) { + // Default filter for CI: exclude ROM-dependent and E2E tests + ::testing::GTEST_FLAG(filter) = "-*RomTest*:-*E2E*:-*ZSCustomOverworld*"; + } + + std::cout << "Running YAZE tests in CI mode..." << std::endl; + std::cout << "Test filter: " << ::testing::GTEST_FLAG(filter) << std::endl; + + // Run tests + int result = RUN_ALL_TESTS(); + + return result; +}