Mirrors
/
open-goal-jak-project
mirror of https://github.com/open-goal/jak-project.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tools: Fix the memory dump analysis tool (#1690) 

* stash

* tools: memory dump working for jak 1 again

* tools: memory dump working for jak 2, kinda hacky?
pull/1697/head
Tyler Wilding 2022-07-22 11:55:18 -04:00 committed by GitHub
parent 1012020035
commit 6ce1d2a0c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 185 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.vs/launch.vs.json
View File

@ -151,8 +151,15 @@
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
"name": "Tools - EE Memory Analyze",
"args": ["${workspaceRoot}/eeMemory.bin", "${workspaceRoot}"]
"name": "Tools - EE Memory Analyze - Jak 1",
"args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak1"]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
"name": "Tools - EE Memory Analyze - Jak 2",
"args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak2"]
},
{
"type": "default",

2
LICENSE
View File

@ -1,6 +1,6 @@
ISC License
Copyright (c) 2020-2021 OpenGOAL Team
Copyright (c) 2020-2022 OpenGOAL Team
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above

3
common/goal_constants.h
View File

@ -18,6 +18,9 @@ constexpr int ARRAY_DATA_OFFSET = 12; // not including type tag
// constexpr s32 SYM_INFO_OFFSET = 8167 * 8 - 4;
namespace jak1 {
constexpr s32 ORIGINAL_MAX_GOAL_SYMBOLS = 8192;
constexpr s32 ORIGINAL_SYM_TO_STRING_OFFSET = 0xff38;
constexpr s32 GOAL_MAX_SYMBOLS = 16384;
constexpr s32 SYM_INFO_OFFSET = GOAL_MAX_SYMBOLS * 8 - 4;

2
tools/CMakeLists.txt
View File

@ -9,6 +9,6 @@ add_executable(dgo_packer
target_link_libraries(dgo_packer common)
add_executable(memory_dump_tool
MemoryDumpTool/main.cpp)
memory_dump_tool/main.cpp)
target_link_libraries(memory_dump_tool common decomp)

262
tools/MemoryDumpTool/main.cpp → tools/memory_dump_tool/main.cpp
View File

@ -4,6 +4,7 @@
#include <string>
#include "common/goal_constants.h"
#include "common/log/log.h"
#include "common/symbols.h"
#include "common/type_system/TypeSystem.h"
#include "common/util/Assert.h"
@ -12,13 +13,10 @@
#include "decompiler/util/DecompilerTypeSystem.h"
#include "third-party/CLI11.hpp"
#include "third-party/fmt/core.h"
#include "third-party/json.hpp"
namespace fs = fs;
constexpr GameVersion kGameVersion = GameVersion::Jak1;
struct Ram {
const u8* data = nullptr;
u32 size = 0;
@ -79,32 +77,62 @@ struct Ram {
bool word_in_memory(u32 addr) const { return in_memory<u32>(addr); }
};
u32 scan_for_symbol_table(const Ram& ram, u32 start_addr, u32 end_addr) {
u32 scan_for_symbol_table(const Ram& ram,
const GameVersion& game_version,
u32 start_addr,
u32 end_addr) {
fmt::print("scanning for symbol table in 0x{:x} - 0x{:x}\n", start_addr, end_addr);
std::vector<u32> candidates;
// look for the false symbol.
for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 8) {
if (ram.word(addr + 4) == addr + 4) {
candidates.push_back(addr);
if (game_version == GameVersion::Jak1) {
for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 8) {
if (ram.word(addr + 4) == addr + 4) {
candidates.push_back(addr);
lg::info("candidate 0x{:x}", addr);
}
}
} else {
for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 4) {
if (ram.word(addr) == addr + 1) {
candidates.push_back(addr + 1);
lg::info("candidate 0x{:x}", addr + 1);
}
}
}
fmt::print("got {} candidates for #f:\n", candidates.size());
for (auto addr : candidates) {
// todo: this is wrong
auto str = addr + BASIC_OFFSET + jak1::SYM_INFO_OFFSET;
fmt::print(" trying 0x{:x}:\n", addr);
if (ram.word_in_memory(str)) {
auto mem = ram.word(str + 4); // offset of str in SymInfo
auto name = ram.try_string(mem + 4); // offset of data in GOAL string
if (name) {
fmt::print(" name: {}\n", *name);
if (game_version == GameVersion::Jak1) {
for (auto addr : candidates) {
auto str = addr + jak1::ORIGINAL_SYM_TO_STRING_OFFSET;
fmt::print(" trying 0x{:x}:\n", addr);
if (ram.word_in_memory(str)) {
auto mem = ram.word(str + 4); // offset of str in SymInfo
auto name = ram.try_string(mem + 4); // offset of data in GOAL string
if (name) {
fmt::print(" name: {}\n", *name);
}
if (name == "#f") {
fmt::print("Got #f = 0x{:x}!\n", addr + 4);
return addr + 4;
}
}
if (name == "#f") {
fmt::print("Got #f = 0x{:x}!\n", addr + 4);
return addr + 4;
}
} else {
for (auto addr : candidates) {
auto str = addr + jak2::SYM_TO_STRING_OFFSET;
fmt::print(" trying 0x{:x}:\n", addr);
if (ram.word_in_memory(str)) {
auto mem = ram.word(str); // offset of str in SymInfo
auto name = ram.try_string(mem + 4); // offset of data in GOAL string
if (name) {
fmt::print(" name: {}\n", *name);
}
if (name == "#f") {
fmt::print("Got #f = 0x{:x}!\n", addr + 4);
return addr;
}
}
}
}
@ -118,68 +146,94 @@ struct SymbolMap {
std::unordered_map<u32, std::string> addr_to_name;
};
SymbolMap build_symbol_map(const Ram& ram, u32 s7) {
// TODO jak 1 specific
fmt::print("finding symbols...\n");
SymbolMap build_symbol_map(const GameVersion& game_version, const Ram& ram, u32 s7) {
lg::info("building symbol map...");
SymbolMap map;
/*
s7 = symbol_table + (GOAL_MAX_SYMBOLS / 2) * 8 + BASIC_OFFSET;
// pointer to the first symbol (SymbolTable2 is the "lower" symbol table)
SymbolTable2 = symbol_table + BASIC_OFFSET;
// the last symbol we will ever access.
LastSymbol = symbol_table + 0xff00;
*/
// todo wrong
auto symbol_table = s7 - ((jak1::GOAL_MAX_SYMBOLS / 2) * 8 + BASIC_OFFSET);
auto SymbolTable2 = symbol_table + BASIC_OFFSET;
auto LastSymbol = symbol_table + 0xff00;
if (game_version == GameVersion::Jak1) {
auto addr_start_of_sym_table = s7 - ((jak1::ORIGINAL_MAX_GOAL_SYMBOLS / 2) * 8);
auto addr_last_symbol = addr_start_of_sym_table + 0xff00;
for (u32 sym = SymbolTable2; sym < LastSymbol; sym += 8) {
auto info = sym + jak1::SYM_INFO_OFFSET; // already has basic offset
auto str = ram.word(info + 4);
if (str) {
auto name = ram.string(str + 4);
if (name != "asize-of-basic-func") {
ASSERT(map.name_to_addr.find(name) == map.name_to_addr.end());
map.name_to_addr[name] = sym;
map.addr_to_name[sym] = name;
map.name_to_value[name] = ram.word(sym);
for (u32 sym = addr_start_of_sym_table; sym < addr_last_symbol; sym += 8) {
auto info = sym + jak1::ORIGINAL_SYM_TO_STRING_OFFSET; // already has basic offset
auto str = ram.word(info);
if (str) {
auto name = ram.string(str + 4);
if (name != "asize-of-basic-func") {
ASSERT(map.name_to_addr.find(name) == map.name_to_addr.end());
map.name_to_addr[name] = sym;
map.addr_to_name[sym] = name;
map.name_to_value[name] = ram.word(sym);
}
}
}
} else {
auto addr_start_of_sym_table = s7 - ((jak2::GOAL_MAX_SYMBOLS / 2) * 4) + BASIC_OFFSET;
auto addr_last_symbol = addr_start_of_sym_table + 0xff00; // ? the same ?
for (u32 sym = addr_start_of_sym_table; sym < addr_last_symbol; sym += 4) {
auto info = sym + jak2::SYM_TO_STRING_OFFSET;
auto str = ram.word(info);
if (str) {
auto name = ram.string(str + 4);
if (name != "asize-of-basic-func") {
ASSERT(map.name_to_addr.find(name) == map.name_to_addr.end());
map.name_to_addr[name] = sym;
map.addr_to_name[sym] = name;
map.name_to_value[name] = ram.word(sym);
}
}
}
}
ASSERT(map.name_to_addr.size() == map.addr_to_name.size());
fmt::print("found {} symbols.\n", map.name_to_addr.size());
lg::info("found {} symbols", map.name_to_addr.size());
return map;
}
std::unordered_map<u32, std::string> build_type_map(const Ram& ram,
const SymbolMap& symbols,
const GameVersion& game_version,
u32 s7) {
// TODO jak 1 specific
std::unordered_map<u32, std::string> result;
fmt::print("finding types...\n");
u32 type_of_type = ram.word(s7 + jak1_symbols::FIX_SYM_TYPE_TYPE);
ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type")));
lg::info("finding types...");
if (game_version == GameVersion::Jak1) {
u32 type_of_type = ram.word(s7 + jak1_symbols::FIX_SYM_TYPE_TYPE);
ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type")));
for (const auto& [name, addr] : symbols.name_to_addr) {
u32 value = ram.word(addr);
if (ram.word_in_memory(value - 4) && ((value & 0x7) == BASIC_OFFSET)) {
if (ram.word(value - 4) == type_of_type) {
result[value] = name;
for (const auto& [name, addr] : symbols.name_to_addr) {
u32 value = ram.word(addr);
if (ram.word_in_memory(value - 4) && ((value & 0x7) == BASIC_OFFSET)) {
if (ram.word(value - 4) == type_of_type) {
result[value] = name;
}
}
}
} else {
u32 type_of_type = ram.word(s7 + jak2_symbols::FIX_SYM_TYPE_TYPE - 1);
ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type") - 1));
for (const auto& [name, addr] : symbols.name_to_addr) {
u32 value = ram.word(addr - 1);
if (ram.word_in_memory(value - 4) && ((value & 0x7) == BASIC_OFFSET)) {
if (ram.word(value - 4) == type_of_type) {
result[value] = name;
}
}
}
}
fmt::print("found {} types\n", result.size());
lg::info("found {} types", result.size());
return result;
}
const std::vector<std::string> ignored_types = {"symbol", "string", "function", "object",
"integer"};
std::unordered_map<std::string, std::vector<u32>> find_basics(
const Ram& ram,
const std::unordered_map<u32, std::string>& type_map) {
fmt::print("Scanning memory for objects. This may take a while...\n");
lg::info("Scanning memory for objects. This may take a while...");
std::unordered_map<std::string, std::vector<u32>> result;
int total_objects = 0;
@ -188,14 +242,14 @@ std::unordered_map<std::string, std::vector<u32>> find_basics(
u32 tag = ram.word(addr);
auto iter = type_map.find(tag);
// ignore the stupid types.
if (iter != type_map.end() && iter->second != "symbol" && iter->second != "string" &&
iter->second != "function" && iter->second != "object" && iter->second != "integer") {
if (iter != type_map.end() && std::find(ignored_types.begin(), ignored_types.end(),
iter->second) == ignored_types.end()) {
result[iter->second].push_back(addr);
total_objects++;
}
}
fmt::print("Got {} objects of {} unique types\n", total_objects, result.size());
lg::info("Got {} objects of {} unique types\n", total_objects, result.size());
return result;
}
@ -563,66 +617,92 @@ int main(int argc, char** argv) {
exit(EXIT_FAILURE);
}
fmt::print("MemoryDumpTool\n");
fs::path dump_path;
fs::path output_path;
std::string game_name = "jak1";
if (argc != 2 && argc != 3) {
fmt::print("usage: memory_dump_tool <ee_ram.bin|savestate.p2s> [output folder]\n");
lg::initialize();
CLI::App app{"OpenGOAL Memory Dump Analyzer"};
app.add_option("dump-path", dump_path, "The path to the dump file to analyze")->required();
app.add_option("--output-path", output_path,
"Where the output files should be sent, defaults to current directory otherwise");
app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
app.validate_positionals();
CLI11_PARSE(app, argc, argv);
auto ok = file_util::setup_project_path({});
if (!ok) {
lg::error("couldn't setup project path, exiting");
return 1;
}
lg::info("Loading type definitions from all-types.gc...");
auto game_version = game_name_to_version(game_name);
decompiler::DecompilerTypeSystem dts(game_version);
// TODO - this could be better
if (game_version == GameVersion::Jak1) {
dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
} else if (game_version == GameVersion::Jak2) {
dts.parse_type_defs({"decompiler", "config", "jak2", "all-types.gc"});
} else {
lg::error("unsupported game version");
return 1;
}
fmt::print("Loading type definitions from all-types.gc...\n");
decompiler::DecompilerTypeSystem dts(kGameVersion);
dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
std::string file_name = argv[1];
fs::path output_folder;
if (!fs::exists(output_folder) || argc < 3) {
fmt::print("Output folder not found, defaulting to current directory");
output_folder = ".";
} else {
output_folder = argv[2];
if (output_folder.empty() || !fs::exists(output_folder)) {
lg::warn("Output folder not found or not provided, defaulting to current directory");
output_folder = "./";
}
if (ends_with(file_name, "p2s")) {
fmt::print("PS2 savestates are not directly supported. Please extract contents beforehand.\n");
if (dump_path.extension() == "p2s") {
lg::error("PCSX2 savestates are not directly supported. Please extract contents beforehand");
return 1;
}
fmt::print("Loading memory from '{}'\n", file_name);
auto data = file_util::read_binary_file(file_name);
lg::info("Loading memory from '{}'", dump_path.string());
auto data = file_util::read_binary_file(dump_path);
u32 one_mb = (1 << 20);
if (data.size() == 32 * one_mb) {
fmt::print("Got 32MB file\n");
lg::info("Got 32MB file");
} else if (data.size() == 128 * one_mb) {
fmt::print("Got 128MB file\n");
lg::info("Got 128MB file");
} else if (data.size() == 127 * one_mb) {
fmt::print("Got a 127MB file. Assuming this is a dump with the first 1 MB missing.\n");
lg::warn("Got a 127MB file. Assuming this is a dump with the first 1 MB missing.\n");
data.insert(data.begin(), one_mb, 0);
ASSERT(data.size() == 128 * one_mb);
if (data.size() != 128 * one_mb) {
lg::error("it was not!");
return 1;
}
} else {
fmt::print("Invalid size: {} bytes\n", data.size());
lg::error("Invalid size: {} bytes", data.size());
return 1;
}
Ram ram(data.data(), data.size());
u32 s7 = scan_for_symbol_table(ram, one_mb, 2 * one_mb);
u32 s7 = scan_for_symbol_table(ram, game_version, one_mb, 2 * one_mb);
if (!s7) {
fmt::print("Failed to find symbol table\n");
lg::error("Failed to find symbol table");
return 1;
}
nlohmann::json results;
if (fs::exists(output_folder / "ee-results.json")) {
fmt::print("Found existing result file, appending results to it!\n");
std::ifstream i(output_folder / "ee-results.json");
auto json_path = fmt::format("ee-results-{}.json", game_name);
if (fs::exists(output_folder / json_path)) {
lg::info("Found existing result file, appending results to it!");
std::ifstream i(output_folder / json_path);
i >> results;
}
auto symbol_map = build_symbol_map(ram, s7);
auto types = build_type_map(ram, symbol_map, s7);
auto symbol_map = build_symbol_map(game_version, ram, s7);
auto types = build_type_map(ram, symbol_map, game_version, s7);
auto basics = find_basics(ram, types);
follow_references_to_find_pointers(ram, dts.ts, basics, s7 + 0x100);
@ -631,10 +711,10 @@ int main(int argc, char** argv) {
inspect_symbols(ram, types, symbol_map);
inspect_process_self(ram, basics, types, dts.ts);
if (fs::exists(output_folder / "ee-results.json")) {
fs::remove(output_folder / "ee-results.json");
if (fs::exists(output_folder / json_path)) {
fs::remove(output_folder / json_path);
}
std::ofstream o(output_folder / "ee-results.json");
std::ofstream o(output_folder / json_path);
o << std::setw(2) << results << std::endl;
return 0;