[Tools] Add DGO packer and unpacker (#219)

* add dgo tools

* make codacy happy

CMakeLists.txt

@@ -76,6 +76,9 @@ add_subdirectory(game)
 # build the compiler
 add_subdirectory(goalc)
 
+# build standalone tools
+add_subdirectory(tools)
+
 # build the gtest libraries
 add_subdirectory(third-party/googletest)
 

common/CMakeLists.txt

@@ -14,6 +14,8 @@ add_library(common
         type_system/TypeFieldLookup.cpp
         type_system/TypeSpec.cpp
         type_system/TypeSystem.cpp
+        util/dgo_util.cpp
+        util/DgoReader.cpp
         util/DgoWriter.cpp
         util/FileUtil.cpp
         util/Timer.cpp

common/util/BinaryReader.h

@@ -11,32 +11,26 @@
 
 class BinaryReader {
  public:
-  BinaryReader(uint8_t* _buffer, uint32_t _size) : buffer(_buffer), size(_size) {}
-
-  explicit BinaryReader(std::vector<uint8_t>& _buffer)
-      : buffer((uint8_t*)_buffer.data()), size(_buffer.size()) {}
+  explicit BinaryReader(const std::vector<uint8_t>& _buffer) : buffer(_buffer) {}
 
   template <typename T>
   T read() {
-    assert(seek + sizeof(T) <= size);
-    T& obj = *(T*)(buffer + seek);
+    assert(seek + sizeof(T) <= buffer.size());
+    T& obj = *(T*)(buffer.data() + seek);
     seek += sizeof(T);
     return obj;
   }
 
   void ffwd(int amount) {
     seek += amount;
-    assert(seek <= size);
+    assert(seek <= buffer.size());
   }
 
-  uint32_t bytes_left() const { return size - seek; }
-
-  uint8_t* here() { return buffer + seek; }
-
-  uint32_t get_seek() { return seek; }
+  uint32_t bytes_left() const { return buffer.size() - seek; }
+  uint8_t* here() { return buffer.data() + seek; }
+  uint32_t get_seek() const { return seek; }
 
  private:
-  uint8_t* buffer;
-  uint32_t size;
+  std::vector<u8> buffer;
   uint32_t seek = 0;
 };

common/util/DgoReader.cpp

@@ -0,0 +1,54 @@
+#include <cstring>
+#include <utility>
+#include <unordered_set>
+#include "DgoReader.h"
+#include "BinaryReader.h"
+#include "common/link_types.h"
+#include "third-party/json.hpp"
+#include "dgo_util.h"
+
+DgoReader::DgoReader(std::string file_name, const std::vector<u8>& data)
+    : m_file_name(std::move(file_name)) {
+  BinaryReader reader(data);
+  auto header = reader.read<DgoHeader>();
+  m_internal_name = header.name;
+  std::unordered_set<std::string> all_unique_names;
+
+  // get all obj files...
+  for (uint32_t i = 0; i < header.object_count; i++) {
+    auto obj_header = reader.read<ObjectHeader>();
+    assert(reader.bytes_left() >= obj_header.size);
+    assert_string_empty_after(obj_header.name, 60);
+
+    DgoDataEntry entry;
+    entry.internal_name = obj_header.name;
+    entry.unique_name = get_object_file_name(entry.internal_name, reader.here(), obj_header.size);
+    all_unique_names.insert(entry.unique_name);
+    entry.data.resize(obj_header.size);
+
+    assert((reader.get_seek() % 16) == 0);
+    memcpy(entry.data.data(), reader.here(), obj_header.size);
+    m_entries.push_back(entry);
+
+    reader.ffwd(obj_header.size);
+  }
+
+  // check we're at the end
+  assert(0 == reader.bytes_left());
+  assert(all_unique_names.size() == m_entries.size());
+}
+
+std::string DgoReader::description_as_json() const {
+  using namespace nlohmann;
+  json j;
+  j["file_name"] = m_file_name;
+  j["internal_name"] = m_internal_name;
+  for (auto& entry : m_entries) {
+    json entry_desc;
+    entry_desc["unique_name"] = entry.unique_name;
+    entry_desc["internal_name"] = entry.internal_name;
+    j["objects"].push_back(entry_desc);
+  }
+
+  return j.dump(4);
+}

common/util/DgoReader.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include "common/common_types.h"
+
+struct DgoDataEntry {
+  std::vector<u8> data;
+  std::string internal_name;
+  std::string unique_name;
+};
+
+class DgoReader {
+ public:
+  DgoReader(std::string file_name, const std::vector<u8>& data);
+  const std::vector<DgoDataEntry> entries() const { return m_entries; }
+  std::string description_as_json() const;
+
+ private:
+  std::vector<DgoDataEntry> m_entries;
+  std::string m_internal_name, m_file_name;
+};

common/util/dgo_util.cpp

@@ -0,0 +1,53 @@
+#include <cassert>
+#include <cstring>
+#include "dgo_util.h"
+#include "common/versions.h"
+#include "third-party/fmt/core.h"
+
+/*!
+ * Assert false if the char[] has non-null data after the null terminated string.
+ * Used to sanity check the sizes of strings in DGO/object file headers.
+ */
+void assert_string_empty_after(const char* str, int size) {
+  auto ptr = str;
+  while (*ptr)
+    ptr++;
+  while (ptr - str < size) {
+    assert(!*ptr);
+    ptr++;
+  }
+}
+
+std::string get_object_file_name(const std::string& original_name, u8* data, int size) {
+  const std::string art_group_text =
+      fmt::format("/src/next/data/art-group{}/",
+                  versions::ART_FILE_VERSION);  // todo, this may change in other games
+  const std::string suffix = "-ag.go";
+
+  int len = int(art_group_text.length());
+  for (int start = 0; start < size; start++) {
+    bool failed = false;
+    for (int i = 0; i < len; i++) {
+      if (start + i >= size || data[start + i] != art_group_text[i]) {
+        failed = true;
+        break;
+      }
+    }
+
+    if (!failed) {
+      for (int i = 0; i < int(original_name.length()); i++) {
+        if (start + len + i >= size || data[start + len + i] != original_name[i]) {
+          assert(false);
+        }
+      }
+
+      assert(int(suffix.length()) + start + len + int(original_name.length()) < size);
+      assert(
+          !memcmp(data + start + len + original_name.length(), suffix.data(), suffix.length() + 1));
+
+      return original_name + "-ag";
+    }
+  }
+
+  return original_name;
+}

common/util/dgo_util.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <string>
+#include "common/common_types.h"
+
+void assert_string_empty_after(const char* str, int size);
+std::string get_object_file_name(const std::string& original_name, u8* data, int size);

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -10,6 +10,8 @@
 #include <set>
 #include <cstring>
 #include <map>
+#include "common/link_types.h"
+#include "common/util/dgo_util.h"
 #include "decompiler/data/tpage.h"
 #include "decompiler/data/game_text.h"
 #include "decompiler/data/StrFileReader.h"
@@ -180,62 +182,6 @@ void ObjectFileDB::load_map_file(const std::string& map_data) {
   }
 }
 
-// Header for a DGO file
-struct DgoHeader {
-  uint32_t size;
-  char name[60];
-};
-
-namespace {
-/*!
- * Assert false if the char[] has non-null data after the null terminated string.
- * Used to sanity check the sizes of strings in DGO/object file headers.
- */
-void assert_string_empty_after(const char* str, int size) {
-  auto ptr = str;
-  while (*ptr)
-    ptr++;
-  while (ptr - str < size) {
-    assert(!*ptr);
-    ptr++;
-  }
-}
-}  // namespace
-
-namespace {
-std::string get_object_file_name(const std::string& original_name, uint8_t* data, int size) {
-  const char art_group_text[] =
-      "/src/next/data/art-group6/";  // todo, this may change in other games
-  const char suffix[] = "-ag.go";
-
-  int len = int(strlen(art_group_text));
-  for (int start = 0; start < size; start++) {
-    bool failed = false;
-    for (int i = 0; i < len; i++) {
-      if (start + i >= size || data[start + i] != art_group_text[i]) {
-        failed = true;
-        break;
-      }
-    }
-
-    if (!failed) {
-      for (int i = 0; i < int(original_name.length()); i++) {
-        if (start + len + i >= size || data[start + len + i] != original_name[i]) {
-          assert(false);
-        }
-      }
-
-      assert(int(strlen(suffix)) + start + len + int(original_name.length()) < size);
-      assert(!memcmp(data + start + len + original_name.length(), suffix, strlen(suffix) + 1));
-
-      return original_name + "-ag";
-    }
-  }
-
-  return original_name;
-}
-}  // namespace
-
 constexpr int MAX_CHUNK_SIZE = 0x8000;
 /*!
  * Load the objects stored in the given DGO into the ObjectFileDB
@@ -303,9 +249,9 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
   assert_string_empty_after(header.name, 60);
 
   // get all obj files...
-  for (uint32_t i = 0; i < header.size; i++) {
+  for (uint32_t i = 0; i < header.object_count; i++) {
     auto obj_header = reader.read<DgoHeader>();
-    assert(reader.bytes_left() >= obj_header.size);
+    assert(reader.bytes_left() >= obj_header.object_count);
     assert_string_empty_after(obj_header.name, 60);
 
     if (std::string(obj_header.name).find("-ag") != std::string::npos) {
@@ -316,10 +262,10 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
       assert(false);
     }
 
-    auto name = get_object_file_name(obj_header.name, reader.here(), obj_header.size);
+    auto name = get_object_file_name(obj_header.name, reader.here(), obj_header.object_count);
 
-    add_obj_from_dgo(name, obj_header.name, reader.here(), obj_header.size, dgo_base_name);
-    reader.ffwd(obj_header.size);
+    add_obj_from_dgo(name, obj_header.name, reader.here(), obj_header.object_count, dgo_base_name);
+    reader.ffwd(obj_header.object_count);
   }
 
   // check we're at the end

tools/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_executable(dgo_unpacker
+        dgo_unpacker.cpp)
+target_link_libraries(dgo_unpacker common)
+
+add_executable(dgo_packer
+        dgo_packer.cpp)
+target_link_libraries(dgo_packer common)

tools/dgo_packer.cpp

@@ -0,0 +1,50 @@
+#include <cstdio>
+#include "common/versions.h"
+#include "common/util/FileUtil.h"
+#include "common/util/BinaryWriter.h"
+#include "third-party/json.hpp"
+
+int main(int argc, char** argv) {
+  printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
+  printf("DGO Packing Tool\n");
+
+  if (argc < 3) {
+    printf("usage: dgo_packer <path> <dgo description file>\n");
+    return 1;
+  }
+
+  std::string out_path = argv[1];
+
+  for (int i = 2; i < argc; i++) {
+    std::string file_name = argv[i];
+    std::string file_text = file_util::read_text_file(file_name);
+
+    auto x = nlohmann::json::parse(file_text);
+    std::string out_file_name = x["file_name"];
+    std::string internal_name = x["internal_name"];
+    printf("Packing %s\n", internal_name.c_str());
+
+    BinaryWriter writer;
+    writer.add<u32>(x["objects"].size());
+    writer.add_cstr_len(x["internal_name"].get<std::string>().c_str(), 60);
+
+    for (auto& entry : x["objects"]) {
+      auto obj_data =
+          file_util::read_binary_file(file_util::combine_path(out_path, entry["unique_name"]));
+      // size
+      writer.add<uint32_t>(obj_data.size());
+      // name
+      writer.add_str_len(entry["internal_name"].get<std::string>().c_str(), 60);
+      // data
+      writer.add_data(obj_data.data(), obj_data.size());
+      // pad
+      while (writer.get_size() & 0xf) {
+        writer.add<uint8_t>(0);
+      }
+    }
+    writer.write_to_file(file_util::combine_path(out_path, "mod_" + out_file_name));
+  }
+
+  printf("Done\n");
+  return 0;
+}

tools/dgo_tools.md

@@ -0,0 +1,11 @@
+## DGO Tools
+The DGO packer and unpacker can be used to extract and repack files in DGOs.
+
+### Unpacking
+Create a folder for the output, then run:
+```tools/dgo_unpacker <path to output> <path to DGO files...>```
+It will then place the files in the output folder. You may specify multiple DGO files, but this is not recommended because sometimes different DGO files have object files with the same name but different data.  It will also create a DGO description file with the same name as the DGO file but with a `.txt` extension.
+
+### Repacking
+```tools/dgo_packer <path to folder with object files> <path to DGO description file>```
+It will repack the DGO. The name will be the same as the original DGO, but with `mod_` in the front.

tools/dgo_unpacker.cpp

@@ -0,0 +1,37 @@
+#include <cstdio>
+#include "common/versions.h"
+#include "common/util/FileUtil.h"
+#include "common/util/DgoReader.h"
+
+int main(int argc, char** argv) {
+  printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
+  printf("DGO Unpacking Tool\n");
+
+  if (argc < 3) {
+    printf("usage: dgo_unpacker <output path> <dgo files>\n");
+    return 1;
+  }
+
+  std::string out_path = argv[1];
+
+  for (int i = 2; i < argc; i++) {
+    std::string file_name = argv[i];
+    std::string base = file_util::base_name(file_name);
+    printf("Unpacking %s\n", base.c_str());
+    // read the file
+    auto data = file_util::read_binary_file(file_name);
+    // read as a DGO
+    auto dgo = DgoReader(base, data);
+    // write dgo description
+    file_util::write_text_file(file_util::combine_path(out_path, base + ".txt"),
+                               dgo.description_as_json());
+    // write files:
+    for (auto& entry : dgo.entries()) {
+      file_util::write_binary_file(file_util::combine_path(out_path, entry.unique_name),
+                                   (void*)entry.data.data(), entry.data.size());
+    }
+  }
+
+  printf("Done\n");
+  return 0;
+}