scawful/yaze
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update ROM compression

Browse Source
This commit is contained in:
Justin Scofield
2022-07-27 22:48:24 -04:00
parent c9e46d1ab8
commit 40d52f02c9
2 changed files with 92 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
src/app/rom.cc
View File

@@ -105,19 +105,17 @@ static char* hexString(const char* str, const unsigned int size) {
return toret; return toret;
} }
static void print_compression_piece(compression_piece* piece) { static void print_CompressionPiece(CompressionPiece* piece) {
printf("Command : %d\n", piece->command); printf("Command : %d\n", piece->command);
printf("length : %d\n", piece->length); printf("length : %d\n", piece->length);
printf("Argument length : %d\n", piece->argument_length); printf("Argument length : %d\n", piece->argument_length);
printf("Argument :%s\n", hexString(piece->argument, piece->argument_length)); printf("Argument :%s\n", hexString(piece->argument, piece->argument_length));
} }
compression_piece* new_compression_piece(const char command, CompressionPiece* NewCompressionPiece(const char command, const int length,
const unsigned int length, const char* args,
const char* args, const int argument_length) {
const unsigned int argument_length) { CompressionPiece* toret = (CompressionPiece*)malloc(sizeof(CompressionPiece));
compression_piece* toret =
(compression_piece*)malloc(sizeof(compression_piece));
toret->command = command; toret->command = command;
toret->length = length; toret->length = length;
if (args != NULL) { if (args != NULL) {
@@ -130,17 +128,22 @@ compression_piece* new_compression_piece(const char command,
return toret; return toret;
} }
void free_compression_chain(compression_piece* piece) { void FreeCompressionPiece(CompressionPiece* piece) {
free(piece->argument);
free(piece);
}
void FreeCompressionChain(CompressionPiece* piece) {
while (piece != NULL) { while (piece != NULL) {
compression_piece* p = piece->next; CompressionPiece* p = piece->next;
free_compression_piece(piece); FreeCompressionPiece(piece);
piece = p; piece = p;
} }
} }
// Merge consecutive copy if possible // Merge consecutive copy if possible
compression_piece* merge_copy(compression_piece* start) { CompressionPiece* MergeCopy(CompressionPiece* start) {
compression_piece* piece = start; CompressionPiece* piece = start;
while (piece != NULL) { while (piece != NULL) {
if (piece->command == kCommandDirectCopy && piece->next != NULL && if (piece->command == kCommandDirectCopy && piece->next != NULL &&
@@ -153,9 +156,9 @@ compression_piece* merge_copy(compression_piece* start) {
memcpy(piece->argument + previous_length, piece->next->argument, memcpy(piece->argument + previous_length, piece->next->argument,
piece->next->argument_length); piece->next->argument_length);
printf("-Merged copy created\n"); printf("-Merged copy created\n");
print_compression_piece(piece); print_CompressionPiece(piece);
compression_piece* p_next_next = piece->next->next; CompressionPiece* p_next_next = piece->next->next;
free_compression_piece(piece->next); FreeCompressionPiece(piece->next);
piece->next = p_next_next; piece->next = p_next_next;
continue; // Next could be another copy continue; // Next could be another copy
} }
@@ -165,10 +168,10 @@ compression_piece* merge_copy(compression_piece* start) {
return start; return start;
} }
unsigned int create_compression_string(compression_piece* start, uchar* output, unsigned int CreateCompressionString(CompressionPiece* start, uchar* output,
char mode) { int mode) {
unsigned int pos = 0; unsigned int pos = 0;
compression_piece* piece = start; CompressionPiece* piece = start;
while (piece != NULL) { while (piece != NULL) {
// Normal header // Normal header
@@ -184,33 +187,35 @@ unsigned int create_compression_string(compression_piece* start, uchar* output,
} else { // We need to split the command } else { // We need to split the command
unsigned int length_left = piece->length - kMaxLengthCompression; unsigned int length_left = piece->length - kMaxLengthCompression;
piece->length = kMaxLengthCompression; piece->length = kMaxLengthCompression;
compression_piece* new_piece = NULL; CompressionPiece* new_piece = NULL;
if (piece->command == kCommandByteFill || if (piece->command == kCommandByteFill ||
piece->command == kCommandWordFill) { piece->command == kCommandWordFill) {
new_piece = new_piece =
new_compression_piece(piece->command, length_left, NewCompressionPiece(piece->command, length_left, piece->argument,
piece->argument, piece->argument_length); piece->argument_length);
} }
if (piece->command == kCommandIncreasingFill) { if (piece->command == kCommandIncreasingFill) {
new_piece = new_piece =
new_compression_piece(piece->command, length_left, NewCompressionPiece(piece->command, length_left, piece->argument,
piece->argument, piece->argument_length); piece->argument_length);
new_piece->argument[0] = new_piece->argument[0] =
(char)(piece->argument[0] + kMaxLengthCompression); (char)(piece->argument[0] + kMaxLengthCompression);
} }
if (piece->command == kCommandDirectCopy) { if (piece->command == kCommandDirectCopy) {
piece->argument_length = kMaxLengthCompression; piece->argument_length = kMaxLengthCompression;
new_piece = new_compression_piece(piece->command, length_left, NULL, new_piece = NewCompressionPiece(piece->command, length_left, NULL,
length_left); length_left);
memcpy(new_piece->argument, piece->argument + kMaxLengthCompression, // MEMCPY
length_left); for (int i = 0; i < length_left; ++i) {
new_piece->argument[i] = piece->argument[i + kMaxLengthCompression];
}
} }
if (piece->command == kCommandRepeatingBytes) { if (piece->command == kCommandRepeatingBytes) {
piece->argument_length = kMaxLengthCompression; piece->argument_length = kMaxLengthCompression;
unsigned int offset = piece->argument[0] + (piece->argument[1] << 8); unsigned int offset = piece->argument[0] + (piece->argument[1] << 8);
new_piece = new_piece =
new_compression_piece(piece->command, length_left, NewCompressionPiece(piece->command, length_left, piece->argument,
piece->argument, piece->argument_length); piece->argument_length);
if (mode == kNintendoMode2) { if (mode == kNintendoMode2) {
new_piece->argument[0] = (offset + kMaxLengthCompression) & 0xFF; new_piece->argument[0] = (offset + kMaxLengthCompression) & 0xFF;
new_piece->argument[1] = (offset + kMaxLengthCompression) >> 8; new_piece->argument[1] = (offset + kMaxLengthCompression) >> 8;
@@ -221,7 +226,7 @@ unsigned int create_compression_string(compression_piece* start, uchar* output,
} }
} }
printf("New added piece\n"); printf("New added piece\n");
print_compression_piece(new_piece); print_CompressionPiece(new_piece);
new_piece->next = piece->next; new_piece->next = piece->next;
piece->next = new_piece; piece->next = new_piece;
continue; continue;
@@ -237,9 +242,15 @@ unsigned int create_compression_string(compression_piece* start, uchar* output,
tmp[0] = piece->argument[1]; tmp[0] = piece->argument[1];
tmp[1] = piece->argument[0]; tmp[1] = piece->argument[0];
} }
memcpy(output + pos, tmp, 2); // memcpy(output + pos, tmp, 2);
for (int i = 0; i < 2; ++i) {
output[pos + i] = tmp[i];
}
} else { } else {
memcpy(output + pos, piece->argument, piece->argument_length); // memcpy(output + pos, piece->argument, piece->argument_length);
for (int i = 0; i < piece->argument_length; ++i) {
output[pos + i] = piece->argument[i];
}
} }
pos += piece->argument_length; pos += piece->argument_length;
piece = piece->next; piece = piece->next;
@@ -248,30 +259,30 @@ unsigned int create_compression_string(compression_piece* start, uchar* output,
return pos + 1; return pos + 1;
} }
absl::StatusOr<Bytes> CompressGraphics(const uint pos, const uint length) { absl::StatusOr<Bytes> ROM::CompressGraphics(const int pos, const int length) {
return Compress(pos, length, kNintendoMode2); return Compress(pos, length, kNintendoMode2);
} }
absl::StatusOr<Bytes> CompressOverworld(const uint pos, const uint length) { absl::StatusOr<Bytes> ROM::CompressOverworld(const int pos, const int length) {
return Compress(pos, length, kNintendoMode1); return Compress(pos, length, kNintendoMode1);
} }
// TODO TEST compressed data border for each cmd // TODO TEST compressed data border for each cmd
absl::StatusOr<Bytes> ROM::Compress(const uint start, const uint length, absl::StatusOr<Bytes> ROM::Compress(const int start, const int length,
char mode) { int mode) {
Bytes compressed_data(length + 10); Bytes compressed_data(length + 10);
// Worse case should be a copy of the string with extended header // Worse case should be a copy of the string with extended header
compression_piece* compressed_chain = new_compression_piece(1, 1, "aaa", 2); CompressionPiece* compressed_chain = NewCompressionPiece(1, 1, "aaa", 2);
compression_piece* compressed_chain_start = compressed_chain; CompressionPiece* compressed_chain_start = compressed_chain;
unsigned int u_data_pos = start; unsigned int u_data_pos = start;
unsigned int last_pos = start + length - 1; unsigned int last_pos = start + length - 1;
printf("max pos :%d\n", last_pos); printf("max pos :%d\n", last_pos);
unsigned int data_size_taken[5] = {0, 0, 0, 0, 0}; unsigned int data_size_taken[5] = {0, 0, 0, 0, 0};
unsigned int cmd_size[5] = {0, 1, 2, 1, 2}; unsigned int cmd_size[5] = {0, 1, 2, 1, 2};
char cmd_args[5][2] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}}; char cmd_args[5][2] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}};
// Used when skipping using copy // Used when skipping using copy
unsigned int bytes_since_last_compression = 0; unsigned int bytes_since_last_compression = 0;
while (1) { while (1) {
memset(data_size_taken, 0, sizeof(data_size_taken)); memset(data_size_taken, 0, sizeof(data_size_taken));
@@ -353,7 +364,7 @@ absl::StatusOr<Bytes> ROM::Compress(const uint start, const uint length,
} }
} }
} }
printf("Finding the best gain\n"); printf("Finding the best gain\n");
// We check if a command managed to pick up 2 or more bytes // We check if a command managed to pick up 2 or more bytes
// We don't want to be even with copy, since it's possible to merge copy // We don't want to be even with copy, since it's possible to merge copy
@@ -372,7 +383,7 @@ absl::StatusOr<Bytes> ROM::Compress(const uint start, const uint length,
max_win = cmd_size_taken; max_win = cmd_size_taken;
} }
} }
// This is the worse case // This is the worse case
if (cmd_with_max == kCommandDirectCopy) { if (cmd_with_max == kCommandDirectCopy) {
printf("- Best command is copy\n"); printf("- Best command is copy\n");
@@ -380,13 +391,14 @@ absl::StatusOr<Bytes> ROM::Compress(const uint start, const uint length,
// something is better after. // something is better after.
u_data_pos++; u_data_pos++;
bytes_since_last_compression++; bytes_since_last_compression++;
// Arbitrary choice to do a 32 bytes grouping // Arbitrary choice to do a 32 bytes grouping
if (bytes_since_last_compression == 32 || u_data_pos > last_pos) { if (bytes_since_last_compression == 32 || u_data_pos > last_pos) {
char buffer[32]; char buffer[32];
memcpy(buffer, u_data + u_data_pos - bytes_since_last_compression, for (int i = 0; i < bytes_since_last_compression; ++i) {
bytes_since_last_compression); buffer[i] = rom_data_[i + u_data_pos - bytes_since_last_compression];
compression_piece* new_comp_piece = new_compression_piece( }
CompressionPiece* new_comp_piece = NewCompressionPiece(
kCommandDirectCopy, bytes_since_last_compression, buffer, kCommandDirectCopy, bytes_since_last_compression, buffer,
bytes_since_last_compression); bytes_since_last_compression);
compressed_chain->next = new_comp_piece; compressed_chain->next = new_comp_piece;
@@ -397,39 +409,37 @@ absl::StatusOr<Bytes> ROM::Compress(const uint start, const uint length,
printf("- Ok we get a gain from %d\n", cmd_with_max); printf("- Ok we get a gain from %d\n", cmd_with_max);
char buffer[2]; char buffer[2];
buffer[0] = cmd_args[cmd_with_max][0]; buffer[0] = cmd_args[cmd_with_max][0];
if (cmd_size[cmd_with_max] == 2) if (cmd_size[cmd_with_max] == 2) buffer[1] = cmd_args[cmd_with_max][1];
buffer[1] = cmd_args[cmd_with_max][1];
CompressionPiece* new_comp_piece = NewCompressionPiece(
compression_piece* new_comp_piece = new_compression_piece(
cmd_with_max, max_win, buffer, cmd_size[cmd_with_max]); cmd_with_max, max_win, buffer, cmd_size[cmd_with_max]);
// If we let non compressed stuff, we need to add a copy chuck before // If we let non compressed stuff, we need to add a copy chuck before
if (bytes_since_last_compression != 0) { if (bytes_since_last_compression != 0) {
char* copy_buff = (char*)malloc(bytes_since_last_compression); char* copy_buff = (char*)malloc(bytes_since_last_compression);
memcpy(copy_buff, u_data + u_data_pos - bytes_since_last_compression, for (int i = 0; i < bytes_since_last_compression; ++i) {
bytes_since_last_compression); copy_buff[i] =
compression_piece* copy_chuck = new_compression_piece( rom_data_[i + u_data_pos - bytes_since_last_compression];
}
CompressionPiece* copy_chuck = NewCompressionPiece(
kCommandDirectCopy, bytes_since_last_compression, copy_buff, kCommandDirectCopy, bytes_since_last_compression, copy_buff,
bytes_since_last_compression); bytes_since_last_compression);
compressed_chain->next = copy_chuck; compressed_chain->next = copy_chuck;
compressed_chain = copy_chuck; compressed_chain = copy_chuck;
} }
compressed_chain->next = new_comp_piece; compressed_chain->next = new_comp_piece;
compressed_chain = new_comp_piece; compressed_chain = new_comp_piece;
u_data_pos += max_win; u_data_pos += max_win;
bytes_since_last_compression = 0; bytes_since_last_compression = 0;
} }
if (u_data_pos > last_pos) break;
if (u_data_pos > last_pos) break;
// Validate compression result // Validate compression result
if (compressed_chain_start->next != NULL) { if (compressed_chain_start->next != NULL) {
// We don't call merge copy so we need more space // We don't call merge copy so we need more space
auto tmp = (uchar*)malloc(length * 2); auto tmp = (uchar*)malloc(length * 2);
auto compressed_size = auto compressed_size =
create_compression_string(compressed_chain_start->next, tmp, mode); CreateCompressionString(compressed_chain_start->next, tmp, mode);
unsigned int p; unsigned int p;
unsigned int k; unsigned int k;
@@ -439,8 +449,8 @@ absl::StatusOr<Bytes> ROM::Compress(const uint start, const uint length,
} }
auto uncomp = std::move(*response); auto uncomp = std::move(*response);
free(tmp); free(tmp);
if (memcmp(uncomp.data(), u_data + start, p) != 0) { if (memcmp(uncomp.data(), rom_data_.data() + start, p) != 0) {
free_compression_chain(compressed_chain_start); FreeCompressionChain(compressed_chain_start);
return absl::InternalError(absl::StrFormat( return absl::InternalError(absl::StrFormat(
"Compressed data does not match uncompressed data at %d\n", "Compressed data does not match uncompressed data at %d\n",
(unsigned int)(u_data_pos - start))); (unsigned int)(u_data_pos - start)));
@@ -448,10 +458,17 @@ absl::StatusOr<Bytes> ROM::Compress(const uint start, const uint length,
} }
} }
// First is a dumb place holder // First is a dumb place holder
merge_copy(compressed_chain_start->next); MergeCopy(compressed_chain_start->next);
auto compressed_size = create_compression_string( uchar temporary_string[length + 10];
compressed_chain_start->next, compressed_data.data(), mode); auto compressed_size = CreateCompressionString(compressed_chain_start->next,
free_compression_chain(compressed_chain_start); temporary_string, mode);
for (int i = 0; i < compressed_size; ++i) {
compressed_data[i] = temporary_string[i];
}
FreeCompressionChain(compressed_chain_start);
for (int i = 0; i < compressed_size; ++i) {
printf("%02x ", compressed_data[i]);
}
return compressed_data; return compressed_data;
} }

30
src/app/rom.h
View File

@@ -35,10 +35,10 @@ constexpr int kMaxLengthNormalHeader = 32;
constexpr int kMaxLengthCompression = 1024; constexpr int kMaxLengthCompression = 1024;
constexpr int kNintendoMode1 = 0; constexpr int kNintendoMode1 = 0;
constexpr int kNintendoMode2 = 1; constexpr int kNintendoMode2 = 1;
constexpr int kTile32Num = 4432;
constexpr uchar kGraphicsBitmap[8] = {0x80, 0x40, 0x20, 0x10, constexpr uchar kGraphicsBitmap[8] = {0x80, 0x40, 0x20, 0x10,
0x08, 0x04, 0x02, 0x01}; 0x08, 0x04, 0x02, 0x01};
static constexpr int kTile32Num = 4432;
using OWBlockset = std::vector<std::vector<ushort>>; using OWBlockset = std::vector<std::vector<ushort>>;
struct OWMapTiles { struct OWMapTiles {
OWBlockset light_world; // 64 maps OWBlockset light_world; // 64 maps
@@ -46,21 +46,13 @@ struct OWMapTiles {
OWBlockset special_world; // 32 maps OWBlockset special_world; // 32 maps
} typedef OWMapTiles; } typedef OWMapTiles;
typedef struct s_compression_piece compression_piece; typedef struct sCompressionPiece CompressionPiece;
struct sCompressionPiece {
struct s_compression_piece {
char command; char command;
unsigned int length; int length;
char* argument; char* argument;
unsigned int argument_length; int argument_length;
compression_piece* next; CompressionPiece* next;
};
struct CompressionPiece {
uchar command;
uint length;
uchar* argument;
uint argument_length;
}; };
class ROM { class ROM {
@@ -69,12 +61,10 @@ class ROM {
absl::Status LoadFromPointer(uchar* data, size_t length); absl::Status LoadFromPointer(uchar* data, size_t length);
absl::Status LoadAllGraphicsData(); absl::Status LoadAllGraphicsData();
// absl::Status SaveOverworld(); absl::StatusOr<Bytes> CompressGraphics(const int pos, const int length);
absl::StatusOr<Bytes> CompressOverworld(const int pos, const int length);
absl::StatusOr<Bytes> CompressGraphics(const uint pos, const uint length); absl::StatusOr<Bytes> Compress(const int start, const int length,
absl::StatusOr<Bytes> CompressOverworld(const uint pos, const uint length); int mode = 0);
absl::StatusOr<Bytes> Compress(const uint start, const uint length,
char mode);
absl::StatusOr<Bytes> DecompressGraphics(int pos, int size); absl::StatusOr<Bytes> DecompressGraphics(int pos, int size);
absl::StatusOr<Bytes> DecompressOverworld(int pos, int size); absl::StatusOr<Bytes> DecompressOverworld(int pos, int size);