Nixpkgs support (#228)

* Use env shebangs

* CMake cleanup, install() invocations, & CTest

`ctest` & the `test` target work now

* Nixpkgs support
pull/221/head
bb010g 2021-02-03 18:29:46 -08:00 committed by GitHub
parent 45f74f078a
commit 2436a8a541
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 332 additions and 97 deletions

8
.gitignore vendored
View File

@ -1,6 +1,12 @@
# for CMake
/Testing
# for clion
cmake-build-debug/*
.idea/*
build/*
decompiler_out/*
logs/*
logs/*
# for Nix
/result*

View File

@ -1,11 +1,8 @@
# Top Level CMakeLists.txt
if (UNIX)
cmake_minimum_required(VERSION 3.10)
else ()
cmake_minimum_required(VERSION 3.16)
endif ()
cmake_minimum_required(VERSION 3.10)
project(jak)
include(CTest)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Debug")
endif()
@ -14,7 +11,7 @@ set(CMAKE_CXX_STANDARD 17)
# Set default compile flags for GCC
# optimization level can be set here. Note that game/ overwrites this for building game C++ code.
if (UNIX)
if(UNIX)
message(STATUS "GCC detected, adding compile flags")
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} \
@ -31,22 +28,22 @@ if (UNIX)
-Wredundant-decls \
-Wshadow \
-Wsign-promo")
else ()
else()
set(CMAKE_CXX_FLAGS "/EHsc")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:10000000")
endif (UNIX)
endif(UNIX)
IF (WIN32)
if(WIN32)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
ENDIF ()
endif()
IF (ASAN_BUILD)
if(ASAN_BUILD)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
message(STATUS "Doing ASAN build")
ENDIF ()
endif()
option(CODE_COVERAGE "Enable Code Coverage Compiler Flags" OFF)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/third-party/cmake/modules/)
@ -80,25 +77,30 @@ add_subdirectory(goalc)
add_subdirectory(tools)
# build the gtest libraries
add_subdirectory(third-party/googletest)
if(WIN32)
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
endif()
add_subdirectory(third-party/googletest EXCLUDE_FROM_ALL)
include(GoogleTest)
# build tests
add_subdirectory(test)
include(test/CMakeLists.txt)
# build minilzo library
add_subdirectory(third-party/minilzo)
add_subdirectory(third-party/minilzo EXCLUDE_FROM_ALL)
# build format library
add_subdirectory(third-party/fmt)
add_subdirectory(third-party/fmt EXCLUDE_FROM_ALL)
# build zydis third party library for disassembling x86
option(ZYDIS_BUILD_TOOLS "" OFF)
option(ZYDIS_BUILD_EXAMPLES "" OFF)
option(ZYDIS_BUILD_SHARED_LIB "" ON)
add_subdirectory("third-party/zydis")
# NOTE: Once under CMake 3.13's policy CMP0077, override with `set()` instead
option(ZYDIS_BUILD_TOOLS "Zydis: Build tools" OFF)
option(ZYDIS_BUILD_EXAMPLES "Zydis: Build examples" OFF)
option(ZYDIS_BUILD_SHARED_LIB "Zydis: Build shared library" ON)
add_subdirectory(third-party/zydis EXCLUDE_FROM_ALL)
# windows memory management lib
IF (WIN32)
if(WIN32)
add_subdirectory(third-party/mman)
ENDIF ()
endif()

View File

@ -37,46 +37,66 @@ We support both Linux and Windows on x86-64.
## Getting Started - Linux (Ubuntu)
Install Packages and Init Repository
Install packages and init repository:
```bash
```sh
sudo apt install gcc make cmake build-essential g++ nasm clang-format
git submodule update --init --recursive
```
Compile
Compile:
```bash
mkdir build && cd build && cmake .. && make -j
```sh
cmake -B build && cmake --build build -j 8
```
Run Tests
Run tests:
```bash
```sh
./test.sh
```
## Getting Started - Linux (Arch)
Install Packages and Init Repository
Install packages and init repository:
```bash
```sh
sudo pacman -S gcc make cmake base-devel g++ nasm
git submodule update --init --recursive
```
Compile
Compile:
```bash
mkdir build && cd build && cmake .. && make -j
```sh
cmake -B build && cmake --build build -j 8
```
Run Tests
Run tests:
```bash
```sh
./test.sh
```
## Getting Started - Nixpkgs
If your Nix supports flakes:
```sh
nix develop # development environment
nix build # package
nix develop '.#jak-asan-dev' # development environment with Clang
nix build '.#jak-asan' # package with Clang ASan build
```
Otherwise, with traditional Nix:
```sh
nix-shell # development environment
nix-build # package
nix-shell -A packages.x86_64-linux.jak-asan-dev # development environment with Clang
nix-build -A packages.x86_64-linux.jak-asan # package with Clang ASan build
```
## Getting Started - Windows
@ -86,13 +106,13 @@ On Windows, it's recommended to get Scoop to use as a package manager, making th
Once Scoop is installed, run the following command:
```ps
```ps1
scoop install llvm nasm
```
Initialize the repository's third-party dependencies:
```bash
```sh
git submodule update --init --recursive
```
@ -204,7 +224,7 @@ Check out these files for more documentation. Some of it is still in progress
## ASan Build
The project supports building with Address Sanitizer (https://github.com/google/sanitizers/wiki/AddressSanitizer) in Linux.
```
```sh
export CXX=clang++
cmake .. -DASAN_BUILD=TRUE
```
@ -215,4 +235,4 @@ You will have to delete the build folder when changing compilers. When running
-- Doing ASAN build
```
Then you can run the tests, runtime, and compiler as normal and they will abort if ASan finds an error.
Then you can run the tests, runtime, and compiler as normal and they will abort if ASan finds an error.

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

View File

@ -1,6 +1,6 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd ${DIR}/out
md5sum --check hash.md5
md5sum --check hash.md5

View File

@ -23,8 +23,10 @@ add_library(common
target_link_libraries(common fmt)
IF(WIN32)
if(WIN32)
target_link_libraries(common wsock32 ws2_32)
ELSE()
else()
target_link_libraries(common stdc++fs)
ENDIF()
endif()
install(TARGETS common)

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

View File

@ -65,3 +65,5 @@ target_link_libraries(decompiler
common
minilzo
fmt)
install(TARGETS decompiler)

4
default.nix Normal file
View File

@ -0,0 +1,4 @@
(import (fetchTarball {
url = "https://github.com/edolstra/flake-compat/archive/99f1c2157fba4bfe6211a321fd0ee43199025dbf.tar.gz";
sha256 = "0x2jn3vrawwv9xp15674wjz9pixwjyj3j771izayl962zziivbx2";
}) { src = ./.; }).defaultNix

25
flake.lock Normal file
View File

@ -0,0 +1,25 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1612312445,
"narHash": "sha256-l8qdZFqwjAGzQrsO+jFqRJA7YnN3jMGe5LGXRMJfcPg=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "6b8fca6b5752c5c6038dc9349d0086cf2c49c567",
"type": "github"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

168
flake.nix Normal file
View File

@ -0,0 +1,168 @@
{
description = "Jak PC ports via OpenGOAL PC port of Naughty Dog's GOAL";
outputs = { self, nixpkgs }: let
getVersionPre = output:
"${nixpkgs.lib.substring 0 8 output.lastModifiedDate}.${output.shortRev or "dirty"}";
supportedSystems = [ "x86_64-linux" "x86_64-darwin" ];
forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system);
supportedPackages = [ "jak" "jak-dev" "jak-asan" "jak-asan-dev" ];
forAllPackages = f: nixpkgs.lib.genAttrs supportedPackages (pname: f pname);
release = false;
version = "0.5.0" + nixpkgs.lib.optionalString (!release) "-${getVersionPre self}";
in {
overlay = pkgs: pkgsSuper: {
jak = pkgs.callPackage (
{ lib, stdenv, buildPackages, fetchFromGitHub, runCommand, writeText
, jak-googletest-src, jak-zydis-src
, llvm ? null
, python3Packages ? null
, enableDevInputs ? false
, enableAsan ? false
, enableFramePointer ? enableAsan
, enablePIE ? false
, enableSourceLevelDebug ? enableAsan
}:
let
sh = lib.escapeShellArg;
optionalFun = b: f: if b then f else x: x;
inherit (stdenv.cc) isClang;
llvm-symbolizer = if isClang then runCommand "llvm-symbolizer" {
allowSubstitutes = false;
preferLocalBuild = true;
} ''
mkdir -p "$out/bin"
cp ${sh (lib.getBin llvm)}/bin/llvm-symbolizer "$out/bin"/llvm-symbolizer
'' else null;
makeFlagArrayBody = prefix: flags:
lib.concatStrings
(lib.mapAttrsToList (n: v: " " + sh "${prefix}${n}=${v}" + " \\\n") flags);
makeFlagArray = name: prefix: flags:
"${name}Array+=( \\\n${makeFlagArrayBody prefix flags})";
in stdenv.mkDerivation {
pname = "jak";
inherit version;
# Workaround until `src = self;` works with Git submodules.
src = runCommand "source" {
allowSubstitutes = false;
preferLocalBuild = true;
src = self;
} ''
cp -R "$src" "$out"
chmod -R u+w "$out"
shopt -s nullglob dotglob
files=("$out/third-party/googletest"/*); if (( ''${#files[*]} == 0 )); then
echo "providing third-party/googletest submodule via Nix"
rm -df "$out/third-party/googletest"
cp -R ${sh jak-googletest-src} "$out/third-party/googletest"
fi
files=("$out/third-party/zydis"/*); if (( ''${#files[*]} == 0 )); then
echo "providing third-party/zydis submodule via Nix"
rm -df "$out/third-party/zydis"
cp -R ${sh jak-zydis-src} "$out/third-party/zydis"
fi
'';
nativeBuildInputs = [
buildPackages.cmake
buildPackages.nasm
] ++ lib.optionals enableDevInputs [
buildPackages.clang-tools # clang-format
python3Packages.pyqt5
python3Packages.python
];
preConfigure = let
cmakeFlags = lib.pipe {
# https://github.com/NixOS/nixpkgs/pull/108496
CMAKE_SKIP_BUILD_RPATH = "OFF";
} [
(optionalFun enableAsan (o: o // {
ASAN_BUILD = "TRUE";
}))
(optionalFun enableFramePointer (o: o // {
CMAKE_C_FLAGS = o.CMAKE_C_FLAGS or "" + " -fno-omit-frame-pointer";
CMAKE_CXX_FLAGS = o.CMAKE_CXX_FLAGS or "" + " -fno-omit-frame-pointer";
}))
(optionalFun enablePIE (o: o // {
POSITION_INDEPENDENT_CODE = "TRUE";
}))
(optionalFun enableSourceLevelDebug (o: o // {
CMAKE_C_FLAGS = o.CMAKE_C_FLAGS or "" + " -g";
CMAKE_CXX_FLAGS = o.CMAKE_CXX_FLAGS or "" + " -g";
}))
];
in ''
${makeFlagArray "cmakeFlags" "-D" cmakeFlags}
'';
doCheck = true;
dontStrip = enableAsan;
preFixup = ''
'' + lib.optionalString (enableAsan && llvm-symbolizer != null) ''
for f in "$out/bin"/*; do
wrapProgram "$f" --set LLVM_SYMBOLIZER ${sh llvm-symbolizer}/bin/llvm-symbolizer
done
'';
meta = with lib; {
description = "OpenGOAL port of Naughty Dog's GOAL";
homepage = "https://github.com/water111/jak-project";
license = lib.licenses.gpl3Plus;
maintainers = with maintainers; [ bb010g ];
platforms = platforms.all;
};
}
) { };
jak-dev = pkgs.jak.override { enableDevInputs = true; };
jak-asan = pkgs.jak.override {
inherit (pkgs.llvmPackages) llvm;
enableAsan = true;
stdenv = pkgs.clangStdenv;
};
jak-asan-dev = pkgs.jak-asan.override { enableDevInputs = true; };
jak-googletest-src = pkgs.callPackage ({ fetchFromGitHub }: fetchFromGitHub {
owner = "google";
repo = "googletest";
rev = "adeef192947fbc0f68fa14a6c494c8df32177508";
sha256 = "1nsl1c5il6mzwggs5fqcp8gyddk9rs6257vlz0zgpik32miq3cgw";
}) { };
jak-zydis-src = pkgs.callPackage ({ fetchFromGitHub }: fetchFromGitHub {
owner = "zyantific";
repo = "zydis";
rev = "c88a4b0cc1271b516b9697af8d088c851745dd60";
sha256 = "1j28vxr3r6w8xawlagcicbd95dcs85shgsm11b98g3qj45bv3haq";
fetchSubmodules = true;
}) { };
};
packages = forAllSystems (system:
let systemPackages = (import nixpkgs {
inherit system;
overlays = [ self.overlay ];
}); in forAllPackages (pname: systemPackages.${pname})
);
defaultPackage = forAllSystems (system:
self.packages.${system}.jak
);
devShell = forAllSystems (system:
self.packages.${system}.jak-dev
);
};
}

View File

@ -74,12 +74,13 @@ set(RUNTIME_SOURCE
add_library(runtime ${RUNTIME_SOURCE})
target_link_libraries(runtime common fmt)
IF (WIN32)
if(WIN32)
target_link_libraries(runtime mman)
ELSE()
else()
target_link_libraries(runtime pthread)
ENDIF()
endif()
add_executable(gk main.cpp)
target_link_libraries(gk runtime)
install(TARGETS gk)

4
gc.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
$DIR/build/goalc/goalc "$@"
$DIR/build/goalc/goalc "$@"

2
gk.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

View File

@ -40,11 +40,11 @@ add_library(compiler
target_link_libraries(compiler common Zydis)
IF (WIN32)
if(WIN32)
target_link_libraries(compiler mman)
ENDIF ()
endif()
add_executable(goalc main.cpp)
target_link_libraries(goalc common Zydis compiler)
install(TARGETS goalc)

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e

View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
import sys
import os

4
shell.nix Normal file
View File

@ -0,0 +1,4 @@
(import (fetchTarball {
url = "https://github.com/edolstra/flake-compat/archive/99f1c2157fba4bfe6211a321fd0ee43199025dbf.tar.gz";
sha256 = "0x2jn3vrawwv9xp15674wjz9pixwjyj3j771izayl962zziivbx2";
}) { src = ./.; }).shellNix

View File

@ -1,6 +1,6 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
$DIR/build/test/goalc-test --gtest_color=yes "$@"
$DIR/build/goalc-test --gtest_color=yes "$@"

View File

@ -1,40 +1,39 @@
set(CMAKE_CXX_STANDARD 17)
include("goalc/CMakeLists.txt")
include(${CMAKE_CURRENT_LIST_DIR}/goalc/CMakeLists.txt)
add_executable(goalc-test
test_main.cpp
test_test.cpp
test_reader.cpp
test_goos.cpp
test_listener_deci2.cpp
test_kernel.cpp
all_jak1_symbols.cpp
test_type_system.cpp
test_CodeTester.cpp
test_emitter.cpp
test_emitter_avx.cpp
test_common_util.cpp
test_pretty_print.cpp
test_zydis.cpp
goalc/test_goal_kernel.cpp
decompiler/FormRegressionTest.cpp
decompiler/test_AtomicOpBuilder.cpp
decompiler/test_FormBeforeExpressions.cpp
decompiler/test_FormExpressionBuild.cpp
decompiler/test_FormExpressionBuildLong.cpp
decompiler/test_InstructionParser.cpp
${CMAKE_CURRENT_LIST_DIR}/test_main.cpp
${CMAKE_CURRENT_LIST_DIR}/test_test.cpp
${CMAKE_CURRENT_LIST_DIR}/test_reader.cpp
${CMAKE_CURRENT_LIST_DIR}/test_goos.cpp
${CMAKE_CURRENT_LIST_DIR}/test_listener_deci2.cpp
${CMAKE_CURRENT_LIST_DIR}/test_kernel.cpp
${CMAKE_CURRENT_LIST_DIR}/all_jak1_symbols.cpp
${CMAKE_CURRENT_LIST_DIR}/test_type_system.cpp
${CMAKE_CURRENT_LIST_DIR}/test_CodeTester.cpp
${CMAKE_CURRENT_LIST_DIR}/test_emitter.cpp
${CMAKE_CURRENT_LIST_DIR}/test_emitter_avx.cpp
${CMAKE_CURRENT_LIST_DIR}/test_common_util.cpp
${CMAKE_CURRENT_LIST_DIR}/test_pretty_print.cpp
${CMAKE_CURRENT_LIST_DIR}/test_zydis.cpp
${CMAKE_CURRENT_LIST_DIR}/goalc/test_goal_kernel.cpp
${CMAKE_CURRENT_LIST_DIR}/decompiler/FormRegressionTest.cpp
${CMAKE_CURRENT_LIST_DIR}/decompiler/test_AtomicOpBuilder.cpp
${CMAKE_CURRENT_LIST_DIR}/decompiler/test_FormBeforeExpressions.cpp
${CMAKE_CURRENT_LIST_DIR}/decompiler/test_FormExpressionBuild.cpp
${CMAKE_CURRENT_LIST_DIR}/decompiler/test_FormExpressionBuildLong.cpp
${CMAKE_CURRENT_LIST_DIR}/decompiler/test_InstructionParser.cpp
${GOALC_TEST_FRAMEWORK_SOURCES}
${GOALC_TEST_CASES})
enable_testing()
target_link_libraries(goalc-test common runtime compiler gtest decomp Zydis)
IF (WIN32)
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
if(WIN32)
target_link_libraries(goalc-test mman)
ENDIF()
endif()
gtest_discover_tests(goalc-test WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
if(UNIX AND CMAKE_COMPILER_IS_GNUCXX AND CODE_COVERAGE)
include(CodeCoverage)

View File

@ -1,12 +1,12 @@
# TODO - probably a more cmakey way to do this
set(GOALC_TEST_CASES
"goalc/all_goalc_template_tests.cpp"
goalc/test_debugger.cpp
goalc/test_game_no_debug.cpp
${CMAKE_CURRENT_LIST_DIR}/all_goalc_template_tests.cpp
${CMAKE_CURRENT_LIST_DIR}/test_debugger.cpp
${CMAKE_CURRENT_LIST_DIR}/test_game_no_debug.cpp
)
set(GOALC_TEST_FRAMEWORK_SOURCES
"goalc/framework/test_runner.cpp"
"goalc/framework/test_runner.h"
${CMAKE_CURRENT_LIST_DIR}/framework/test_runner.cpp
${CMAKE_CURRENT_LIST_DIR}/framework/test_runner.h
)

View File

@ -195,4 +195,4 @@ TEST_F(ControlStatementTests, Desfun) {
TEST_F(ControlStatementTests, DeReference) {
runner->run_static_test(env, testCategory, "methods.static.gc", {"#t#t\n0\n"});
}
}

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

View File

@ -4,4 +4,6 @@ target_link_libraries(dgo_unpacker common)
add_executable(dgo_packer
dgo_packer.cpp)
target_link_libraries(dgo_packer common)
target_link_libraries(dgo_packer common)
install(TARGETS dgo_unpacker dgo_packer)