scawful/yaze
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ValidateCompressionResult, reorganize process.

Browse Source
This commit is contained in:
Justin Scofield
2022-08-03 19:31:17 +00:00
parent 1cd2ba106b
commit 3cd02e97af
2 changed files with 105 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
src/app/rom.cc
View File

@@ -23,7 +23,7 @@
#define OVERWORLD_GRAPHICS_POS_1 0x4F80 #define OVERWORLD_GRAPHICS_POS_1 0x4F80
#define OVERWORLD_GRAPHICS_POS_2 0x505F #define OVERWORLD_GRAPHICS_POS_2 0x505F
#define OVERWORLD_GRAPHICS_POS_3 0x513E #define OVERWORLD_GRAPHICS_POS_3 0x513E
#define COMPRESSION_STRING_MOD 7 << 5 #define COMPRESSION_STRING_MOD 7 << 5
#define SNES_BYTE_MAX 0xFF #define SNES_BYTE_MAX 0xFF
@@ -56,6 +56,16 @@ void PrintCompressionPiece(const std::shared_ptr<CompressionPiece>& piece) {
printf("\nArgument length : %d\n", piece->argument_length); printf("\nArgument length : %d\n", piece->argument_length);
} }
void PrintCompressionChain(
const std::shared_ptr<CompressionPiece>& compressed_chain_start) {
auto compressed_chain = compressed_chain_start->next;
while (compressed_chain != NULL) {
printf("--Piece--\n");
PrintCompressionPiece(compressed_chain);
compressed_chain = compressed_chain->next;
}
}
std::shared_ptr<CompressionPiece> NewCompressionPiece( std::shared_ptr<CompressionPiece> NewCompressionPiece(
const char command, const int length, const std::string args, const char command, const int length, const std::string args,
const int argument_length) { const int argument_length) {
@@ -125,11 +135,13 @@ std::shared_ptr<CompressionPiece> SplitCompressionPiece(
new_piece = NewCompressionPiece(piece->command, length_left, new_piece = NewCompressionPiece(piece->command, length_left,
piece->argument, piece->argument_length); piece->argument, piece->argument_length);
if (mode == kNintendoMode2) { if (mode == kNintendoMode2) {
new_piece->argument[0] = (offset + kMaxLengthCompression) & SNES_BYTE_MAX; new_piece->argument[0] =
(offset + kMaxLengthCompression) & SNES_BYTE_MAX;
new_piece->argument[1] = (offset + kMaxLengthCompression) >> 8; new_piece->argument[1] = (offset + kMaxLengthCompression) >> 8;
} }
if (mode == kNintendoMode1) { if (mode == kNintendoMode1) {
new_piece->argument[1] = (offset + kMaxLengthCompression) & SNES_BYTE_MAX; new_piece->argument[1] =
(offset + kMaxLengthCompression) & SNES_BYTE_MAX;
new_piece->argument[0] = (offset + kMaxLengthCompression) >> 8; new_piece->argument[0] = (offset + kMaxLengthCompression) >> 8;
} }
} break; } break;
@@ -138,18 +150,19 @@ std::shared_ptr<CompressionPiece> SplitCompressionPiece(
} }
Bytes CreateCompressionString(std::shared_ptr<CompressionPiece>& start, Bytes CreateCompressionString(std::shared_ptr<CompressionPiece>& start,
int mode) { int mode) {
uint pos = 0; uint pos = 0;
auto piece = start; auto piece = start;
Bytes output; Bytes output;
while (piece != nullptr) { while (piece != nullptr) {
if (piece->length <= kMaxLengthNormalHeader) { // Normal header if (piece->length <= kMaxLengthNormalHeader) { // Normal header
output.push_back(BUILD_HEADER(piece->command, piece->length)); output.push_back(BUILD_HEADER(piece->command, piece->length));
pos++; pos++;
} else { } else {
if (piece->length <= kMaxLengthCompression) { if (piece->length <= kMaxLengthCompression) {
output.push_back((COMPRESSION_STRING_MOD) | ((uchar)piece->command << 2) | output.push_back((COMPRESSION_STRING_MOD) |
((uchar)piece->command << 2) |
(((piece->length - 1) & 0xFF00) >> 8)); (((piece->length - 1) & 0xFF00) >> 8));
pos++; pos++;
printf("Building extended header : cmd: %d, length: %d - %02X\n", printf("Building extended header : cmd: %d, length: %d - %02X\n",
@@ -196,10 +209,10 @@ Bytes CreateCompressionString(std::shared_ptr<CompressionPiece>& start,
// Test every command to see the gain with current position // Test every command to see the gain with current position
void TestAllCommands(const uchar* rom_data, DataSizeArray& data_size_taken, void TestAllCommands(const uchar* rom_data, DataSizeArray& data_size_taken,
CommandArgumentArray& cmd_args, uint& u_data_pos, CommandArgumentArray& cmd_args, uint& src_data_pos,
const uint last_pos, uint start) { const uint last_pos, uint start) {
{ // BYTE REPEAT { // BYTE REPEAT
uint pos = u_data_pos; uint pos = src_data_pos;
char byte_to_repeat = rom_data[pos]; char byte_to_repeat = rom_data[pos];
while (pos <= last_pos && rom_data[pos] == byte_to_repeat) { while (pos <= last_pos && rom_data[pos] == byte_to_repeat) {
data_size_taken[kCommandByteFill]++; data_size_taken[kCommandByteFill]++;
@@ -208,9 +221,9 @@ void TestAllCommands(const uchar* rom_data, DataSizeArray& data_size_taken,
cmd_args[kCommandByteFill][0] = byte_to_repeat; cmd_args[kCommandByteFill][0] = byte_to_repeat;
} }
{ // WORD REPEAT { // WORD REPEAT
if (u_data_pos + 2 <= last_pos && if (src_data_pos + 2 <= last_pos &&
rom_data[u_data_pos] != rom_data[u_data_pos + 1]) { rom_data[src_data_pos] != rom_data[src_data_pos + 1]) {
uint pos = u_data_pos; uint pos = src_data_pos;
char byte1 = rom_data[pos]; char byte1 = rom_data[pos];
char byte2 = rom_data[pos + 1]; char byte2 = rom_data[pos + 1];
pos += 2; pos += 2;
@@ -227,7 +240,7 @@ void TestAllCommands(const uchar* rom_data, DataSizeArray& data_size_taken,
} }
} }
{ // INC BYTE { // INC BYTE
uint pos = u_data_pos; uint pos = src_data_pos;
char byte = rom_data[pos]; char byte = rom_data[pos];
pos++; pos++;
data_size_taken[kCommandIncreasingFill] = 1; data_size_taken[kCommandIncreasingFill] = 1;
@@ -235,23 +248,23 @@ void TestAllCommands(const uchar* rom_data, DataSizeArray& data_size_taken,
data_size_taken[kCommandIncreasingFill]++; data_size_taken[kCommandIncreasingFill]++;
pos++; pos++;
} }
cmd_args[kCommandIncreasingFill][0] = rom_data[u_data_pos]; cmd_args[kCommandIncreasingFill][0] = rom_data[src_data_pos];
} }
{ // INTRA CPY { // INTRA CPY
if (u_data_pos != start) { if (src_data_pos != start) {
uint searching_pos = start; uint searching_pos = start;
uint current_pos_u = u_data_pos; uint current_pos_u = src_data_pos;
uint copied_size = 0; uint copied_size = 0;
uint search_start = start; uint search_start = start;
while (searching_pos < u_data_pos && current_pos_u <= last_pos) { while (searching_pos < src_data_pos && current_pos_u <= last_pos) {
while (rom_data[current_pos_u] != rom_data[searching_pos] && while (rom_data[current_pos_u] != rom_data[searching_pos] &&
searching_pos < u_data_pos) searching_pos < src_data_pos)
searching_pos++; searching_pos++;
search_start = searching_pos; search_start = searching_pos;
while (current_pos_u <= last_pos && while (current_pos_u <= last_pos &&
rom_data[current_pos_u] == rom_data[searching_pos] && rom_data[current_pos_u] == rom_data[searching_pos] &&
searching_pos < u_data_pos) { searching_pos < src_data_pos) {
copied_size++; copied_size++;
current_pos_u++; current_pos_u++;
searching_pos++; searching_pos++;
@@ -263,7 +276,7 @@ void TestAllCommands(const uchar* rom_data, DataSizeArray& data_size_taken,
cmd_args[kCommandRepeatingBytes][0] = search_start & SNES_BYTE_MAX; cmd_args[kCommandRepeatingBytes][0] = search_start & SNES_BYTE_MAX;
cmd_args[kCommandRepeatingBytes][1] = search_start >> 8; cmd_args[kCommandRepeatingBytes][1] = search_start >> 8;
} }
current_pos_u = u_data_pos; current_pos_u = src_data_pos;
copied_size = 0; copied_size = 0;
} }
} }
@@ -290,34 +303,36 @@ void ValidateForByteGain(const DataSizeArray& data_size_taken,
} }
} }
void CompressionDirectCopy(const uchar* rom_data, absl::Status ValidateCompressionResult(
std::shared_ptr<CompressionPiece>& compressed_chain, std::shared_ptr<CompressionPiece>& compressed_chain_start, int mode,
uint& u_data_pos, uint& bytes_since_last_compression, int start, int src_data_pos) {
uint& last_pos) { if (compressed_chain_start->next != nullptr) {
// We just move through the next byte and don't 'compress' yet, maybe ROM temp_rom;
// something is better after. auto rom_response = temp_rom.LoadFromBytes(
u_data_pos++; CreateCompressionString(compressed_chain_start->next, mode));
bytes_since_last_compression++; if (!rom_response.ok()) {
return rom_response;
// Arbitrary choice to do a 32 bytes grouping }
if (bytes_since_last_compression == 32 || u_data_pos > last_pos) { auto decomp_response = temp_rom.Decompress(0, temp_rom.GetSize());
std::string buffer; if (!decomp_response.ok()) {
for (int i = 0; i < bytes_since_last_compression; ++i) { return decomp_response.status();
buffer.push_back(rom_data[i + u_data_pos - bytes_since_last_compression]); }
auto decomp_data = std::move(*decomp_response);
if (!std::equal(decomp_data.begin() + start, decomp_data.end(),
temp_rom.begin())) {
return absl::InternalError(absl::StrFormat(
"Compressed data does not match uncompressed data at %d\n",
(uint)(src_data_pos - start)));
} }
auto new_comp_piece =
NewCompressionPiece(kCommandDirectCopy, bytes_since_last_compression,
buffer, bytes_since_last_compression);
compressed_chain->next = new_comp_piece;
compressed_chain = new_comp_piece;
bytes_since_last_compression = 0;
} }
return absl::OkStatus();
} }
void CompressionCommandAlternative( void CompressionCommandAlternative(
const uchar* rom_data, std::shared_ptr<CompressionPiece>& compressed_chain, const uchar* rom_data, std::shared_ptr<CompressionPiece>& compressed_chain,
const CommandSizeArray& cmd_size, const CommandArgumentArray& cmd_args, const CommandSizeArray& cmd_size, const CommandArgumentArray& cmd_args,
uint& u_data_pos, uint& bytes_since_last_compression, uint& cmd_with_max, uint& src_data_pos, uint& comp_accumulator, uint& cmd_with_max,
uint& max_win) { uint& max_win) {
printf("- Ok we get a gain from %d\n", cmd_with_max); printf("- Ok we get a gain from %d\n", cmd_with_max);
std::string buffer; std::string buffer;
@@ -330,30 +345,29 @@ void CompressionCommandAlternative(
cmd_size[cmd_with_max]); cmd_size[cmd_with_max]);
PrintCompressionPiece(new_comp_piece); PrintCompressionPiece(new_comp_piece);
// If we let non compressed stuff, we need to add a copy chunk before // If we let non compressed stuff, we need to add a copy chunk before
if (bytes_since_last_compression != 0) { if (comp_accumulator != 0) {
std::string copy_buff; std::string copy_buff;
copy_buff.resize(bytes_since_last_compression); copy_buff.resize(comp_accumulator);
for (int i = 0; i < bytes_since_last_compression; ++i) { for (int i = 0; i < comp_accumulator; ++i) {
copy_buff[i] = rom_data[i + u_data_pos - bytes_since_last_compression]; copy_buff[i] = rom_data[i + src_data_pos - comp_accumulator];
} }
auto copy_chunk = auto copy_chunk = NewCompressionPiece(kCommandDirectCopy, comp_accumulator,
NewCompressionPiece(kCommandDirectCopy, bytes_since_last_compression, copy_buff, comp_accumulator);
copy_buff, bytes_since_last_compression);
compressed_chain->next = copy_chunk; compressed_chain->next = copy_chunk;
compressed_chain = copy_chunk; compressed_chain = copy_chunk;
} else { } else {
compressed_chain->next = new_comp_piece; compressed_chain->next = new_comp_piece;
compressed_chain = new_comp_piece; compressed_chain = new_comp_piece;
} }
u_data_pos += max_win; src_data_pos += max_win;
bytes_since_last_compression = 0; comp_accumulator = 0;
} }
} // namespace } // namespace
// TODO TEST compressed data border for each cmd // TODO TEST compressed data border for each cmd
absl::StatusOr<Bytes> ROM::Compress(const int start, const int length, absl::StatusOr<Bytes> ROM::Compress(const int start, const int length, int mode,
int mode) { bool check) {
// Worse case should be a copy of the string with extended header // Worse case should be a copy of the string with extended header
auto compressed_chain = NewCompressionPiece(1, 1, "aaa", 2); auto compressed_chain = NewCompressionPiece(1, 1, "aaa", 2);
auto compressed_chain_start = compressed_chain; auto compressed_chain_start = compressed_chain;
@@ -362,70 +376,64 @@ absl::StatusOr<Bytes> ROM::Compress(const int start, const int length,
DataSizeArray data_size_taken = {0, 0, 0, 0, 0}; DataSizeArray data_size_taken = {0, 0, 0, 0, 0};
CommandSizeArray cmd_size = {0, 1, 2, 1, 2}; CommandSizeArray cmd_size = {0, 1, 2, 1, 2};
uint u_data_pos = start; uint src_data_pos = start;
uint last_pos = start + length - 1; uint last_pos = start + length - 1;
uint bytes_since_last_compression = 0; // Used when skipping using copy uint comp_accumulator = 0; // Used when skipping using copy
while (1) { while (1) {
data_size_taken.fill({}); data_size_taken.fill({});
cmd_args.fill({{}}); cmd_args.fill({{}});
TestAllCommands(rom_data_.data(), data_size_taken, cmd_args, u_data_pos, TestAllCommands(rom_data_.data(), data_size_taken, cmd_args, src_data_pos,
last_pos, start); last_pos, start);
uint max_win = 2; uint max_win = 2;
uint cmd_with_max = kCommandDirectCopy; uint cmd_with_max = kCommandDirectCopy;
ValidateForByteGain(data_size_taken, cmd_size, max_win, cmd_with_max); ValidateForByteGain(data_size_taken, cmd_size, max_win, cmd_with_max);
if (cmd_with_max == kCommandDirectCopy) { switch (cmd_with_max) {
printf("- Best command is copy\n"); case kCommandDirectCopy: // This is the worst case scenario
// This is the worse case // Progress through the next byte, in case there's a different
CompressionDirectCopy(rom_data_.data(), compressed_chain, u_data_pos, // compression command we can implement before we hit 32 bytes.
bytes_since_last_compression, last_pos); src_data_pos++;
} else { comp_accumulator++;
// Yay we get something better
CompressionCommandAlternative( // Arbitrary choice to do a 32 bytes grouping for copy.
rom_data_.data(), compressed_chain, cmd_size, cmd_args, u_data_pos, if (comp_accumulator == 32 || src_data_pos > last_pos) {
bytes_since_last_compression, cmd_with_max, max_win); std::string buffer;
for (int i = 0; i < comp_accumulator; ++i) {
buffer.push_back(rom_data_[i + src_data_pos - comp_accumulator]);
}
auto new_comp_piece = NewCompressionPiece(
kCommandDirectCopy, comp_accumulator, buffer, comp_accumulator);
compressed_chain->next = new_comp_piece;
compressed_chain = new_comp_piece;
comp_accumulator = 0;
}
break;
default: // Anything is better than directly copying bytes...
CompressionCommandAlternative(rom_data_.data(), compressed_chain,
cmd_size, cmd_args, src_data_pos,
comp_accumulator, cmd_with_max, max_win);
break;
} }
if (u_data_pos > last_pos) { if (src_data_pos > last_pos) {
printf("Breaking compression loop\n"); printf("Breaking compression loop\n");
break; break;
} }
// Validate compression result if (check) {
if (compressed_chain_start->next != nullptr) { auto response = ValidateCompressionResult(compressed_chain_start, mode,
ROM temp_rom; start, src_data_pos);
auto rom_response = temp_rom.LoadFromBytes( if (!response.ok()) {
CreateCompressionString(compressed_chain_start->next, mode)); return response;
if (!rom_response.ok()) {
return rom_response;
}
auto decomp_response = temp_rom.Decompress(0, temp_rom.GetSize());
if (!decomp_response.ok()) {
return decomp_response.status();
}
auto decomp_data = std::move(*decomp_response);
if (!std::equal(decomp_data.begin() + start, decomp_data.end(),
temp_rom.begin())) {
return absl::InternalError(absl::StrFormat(
"Compressed data does not match uncompressed data at %d\n",
(uint)(u_data_pos - start)));
} }
} }
} }
MergeCopy(compressed_chain_start->next); // Skipping compression chain header MergeCopy(compressed_chain_start->next); // Skipping compression chain header
PrintCompressionChain(compressed_chain_start);
compressed_chain = compressed_chain_start->next;
while (compressed_chain != NULL) {
printf("--Piece--\n");
PrintCompressionPiece(compressed_chain);
compressed_chain = compressed_chain->next;
}
return CreateCompressionString(compressed_chain_start->next, mode); return CreateCompressionString(compressed_chain_start->next, mode);
} }
@@ -445,9 +453,11 @@ absl::StatusOr<Bytes> ROM::Decompress(int offset, int size, bool reversed) {
while (databyte != SNES_BYTE_MAX) { // End of decompression while (databyte != SNES_BYTE_MAX) { // End of decompression
databyte = rom_data_[offset]; databyte = rom_data_[offset];
if ((databyte & CMD_EXPANDED_MOD) == CMD_EXPANDED_MOD) { // Expanded Command if ((databyte & CMD_EXPANDED_MOD) ==
CMD_EXPANDED_MOD) { // Expanded Command
cmd = ((databyte >> 2) & CMD_MOD); cmd = ((databyte >> 2) & CMD_MOD);
length = (((databyte << 8) | rom_data_[offset + 1]) & CMD_EXPANDED_LENGTH_MOD); length =
(((databyte << 8) | rom_data_[offset + 1]) & CMD_EXPANDED_LENGTH_MOD);
offset += 2; // Advance 2 bytes in ROM offset += 2; // Advance 2 bytes in ROM
} else { // Normal Command } else { // Normal Command
cmd = ((databyte >> 5) & CMD_MOD); cmd = ((databyte >> 5) & CMD_MOD);

2
src/app/rom.h
View File

@@ -67,7 +67,7 @@ struct OWMapTiles {
class ROM { class ROM {
public: public:
absl::StatusOr<Bytes> Compress(const int start, const int length, absl::StatusOr<Bytes> Compress(const int start, const int length,
int mode = 1); int mode = 1, bool check = false);
absl::StatusOr<Bytes> CompressGraphics(const int pos, const int length); absl::StatusOr<Bytes> CompressGraphics(const int pos, const int length);
absl::StatusOr<Bytes> CompressOverworld(const int pos, const int length); absl::StatusOr<Bytes> CompressOverworld(const int pos, const int length);