initial commit

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+.idea/
+RLHookLib/__pycache__/
+RLHookLib/.idea/
+RLHookLib/cmake-build-debug/
+RLHookLib/bin/
+RLHookLib/build/

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "RLHookLib/libs/pybind11"]
+	path = RLHookLib/libs/pybind11
+	url = https://github.com/pybind/pybind11.git
+[submodule "RLHookLib/libs/Detours/Detours"]
+	path = RLHookLib/libs/Detours/Detours
+	url = https://github.com/microsoft/Detours.git

README.md

@@ -0,0 +1,15 @@
+# Super Hexagon AI
+
+ - Based on Reinforcement Learning (Q-Learning)
+ - Beats all six levels easily end to end (i. e. from pixel input)
+ - Fast C++ implementation that hooks into the game and hands the screen over to the python process
+
+
+# Approach
+This AI is based on Reinforcement Learning more specifically Deep Q-Learning [1]. 
+I. e. the Q-Function is learned which represents the discounted expected future reward if the agent acts according to the Q-Function.
+The agent receives a reward of -1 if he dies otherwise a reward of 0.  
+
+# References
+[1] Mnih, Volodymyr, et al. "Human-level control through deep reinforcement learning." nature 518.7540 (2015): 529-533. 
+

RLHookLib/CMakeLists.txt

@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.10)
+project(RLHookLib)
+
+set(CMAKE_CXX_STANDARD 17)
+
+if (NOT EXISTS "${PROJECT_SOURCE_DIR}/libs/pybind11/CMakeLists.txt" OR NOT EXISTS "${PROJECT_SOURCE_DIR}/libs/Detours/Detours/README.md")
+    message(FATAL_ERROR "The submodules pybind11 and/or Detours have not been cloned! Clone them with \"git submodule update --init --recursive\".")
+endif()
+
+add_subdirectory(libs)
+find_package(OpenGL REQUIRED)
+
+add_subdirectory(RLHookDLL)
+if(NOT DEFINED ONLY_ADDITIONAL_BINARIES)
+    add_subdirectory(PyRLHook)
+endif()
+add_subdirectory(FunctionAddressGetter)
+add_subdirectory(Utils)

RLHookLib/FunctionAddressGetter/CMakeLists.txt

@@ -0,0 +1,4 @@
+
+set(FUNCTION_ADDRESS_GETTER "FunctionAddressGetter")
+
+add_executable(${FUNCTION_ADDRESS_GETTER} FunctionAddressGetter.cpp)

RLHookLib/FunctionAddressGetter/FunctionAddressGetter.cpp

@@ -0,0 +1,10 @@
+#include <iostream>
+#include <Windows.h>
+
+int main(int argc, char *argv[]) {
+    if (argc != 3) {
+        std::cout << "Usage: FunctionAddressGetter.exe [Module] [Function]" << std::endl;
+        return 1;
+    }
+    return reinterpret_cast<int>(GetProcAddress(GetModuleHandle(argv[1]), argv[2]));
+}

RLHookLib/PyRLHook/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(PY_RL_HOOK "pyrlhook")
+
+pybind11_add_module(${PY_RL_HOOK} src/PythonBindings.cpp src/GameInterface.cpp src/Utils.cpp)
+target_link_libraries(${PY_RL_HOOK} PRIVATE pybind11::embed OpenGL::GL Utils)
+target_include_directories(${PY_RL_HOOK} PRIVATE ../Utils)
+

RLHookLib/PyRLHook/src/GameInterface.cpp

@@ -0,0 +1,216 @@
+
+#include "GameInterface.h"
+#include <chrono>
+#include <filesystem>
+#include "Utils.h"
+
+#define OPEN_PROCESS_ACCESS_RIGHTS (PROCESS_CREATE_THREAD | PROCESS_QUERY_INFORMATION | PROCESS_VM_OPERATION | PROCESS_VM_WRITE | PROCESS_VM_READ)
+
+#define RL_HOOK_DLL_NAME std::string("RLHookDLL.dll")
+#define RL_HOOK_DLL_PATH_x86 ("\\bin\\Win32\\" + RL_HOOK_DLL_NAME)
+#define RL_HOOK_DLL_PATH_x64 ("\\bin\\x64\\" + RL_HOOK_DLL_NAME)
+
+#define TIME_DISTORTER_DLL_NAME std::string("TimeDistorter.dll")
+
+#define EXCEPTION_MS_NAME "\\\\.\\mailslot\\rlhook-exception"
+#define SHARED_MEMORY_NAME "Global\\rlhook-sharedmemory"
+#define PIPE_NAME "\\\\.\\pipe\\rlhook-pipe"
+#define TIME_DISTORTER_PIPE_NAME "\\\\.\\pipe\\rlhook-isalivepipe"
+
+std::string GameInterface::basePath;
+
+GameInterface::GameInterface(
+        const std::string &processName,
+        PixelFormat pixelFormat,
+        PixelDataType pixelDataType
+) : GameInterface(Utils::getProcessPid(processName), pixelFormat, pixelDataType) {}
+
+GameInterface::GameInterface(DWORD pid,
+        PixelFormat pixelFormat,
+        PixelDataType pixelDataType
+) : pid(pid), pixelFormat(pixelFormat), pixelDataType(pixelDataType) {
+
+    try {
+
+        // get handle to target process
+        gameProcess = OpenProcess(OPEN_PROCESS_ACCESS_RIGHTS, FALSE, pid);
+        Utils::checkError(gameProcess == nullptr, "OpenProcess");
+
+        // create mailslot to which the target process can communicate exceptions
+        exceptionMailSlot = Utils::createMailslot(EXCEPTION_MS_NAME);
+
+        // inject dll into target and attach pipe
+        bool isx86 = Utils::isWow64Process(gameProcess);
+        std::string dllPath = basePath + (isx86 ? RL_HOOK_DLL_PATH_x86 : RL_HOOK_DLL_PATH_x64);
+
+        if (!std::filesystem::exists(dllPath))
+            throw std::runtime_error(RL_HOOK_DLL_NAME + " wasn't found at \"" + dllPath + "\".");
+
+        HMODULE dllModule = Utils::getModuleBaseAddress(gameProcess, RL_HOOK_DLL_NAME);
+        if (dllModule == nullptr) {
+            Utils::attachDll(gameProcess, dllPath);
+            pipe = Pipe(PIPE_NAME, false);
+        } else {
+            try {
+                pipe = Pipe(PIPE_NAME, false);
+            } catch (const WindowsError& e) {
+                if (e.errorCode != ERROR_FILE_NOT_FOUND)
+                    throw;
+                detachDll();
+                Utils::attachDll(gameProcess, dllPath);
+                pipe = Pipe(PIPE_NAME, false);
+            }
+        }
+
+        // write parameters
+        pipe.write<GLenum>(pixelFormat);
+        pipe.write<GLenum>(pixelDataType);
+
+        // receive parameters
+        width = pipe.read<std::uint32_t>();
+        height = pipe.read<std::uint32_t>();
+        channels = pipe.read<std::uint32_t>();
+        dataTypeSize = pipe.read<std::uint32_t>();
+
+        // wait for init in target process to finish
+        pipe.read<bool>();
+
+        // open shared memory
+        hMapFile = OpenFileMapping(
+                FILE_MAP_READ,
+                FALSE,
+                SHARED_MEMORY_NAME
+        );
+        Utils::checkError(hMapFile == nullptr, "OpenFileMapping");
+
+        pBuf = MapViewOfFile(
+                hMapFile,
+                FILE_MAP_READ,
+                0,
+                0,
+                bufferSize()
+        );
+        Utils::checkError(pBuf == nullptr, "MapViewOfFile");
+
+        // wait for glSwapBuffers to be called for the first time
+        pipe.read<bool>();
+
+        // attach pipe to time distorter
+        timeDistorterPipe = Pipe(TIME_DISTORTER_PIPE_NAME, false);
+    } catch (...) {
+        if (exceptionMailSlot != INVALID_HANDLE_VALUE)
+            checkForException();
+        throw;
+    }
+}
+
+GameInterface::~GameInterface() {
+    finish();
+}
+
+void GameInterface::finish() {
+    isFinished = true;
+
+    pipe = Pipe();
+    timeDistorterPipe = Pipe();
+
+    UnmapViewOfFile(pBuf);
+    CloseHandle(hMapFile);
+    CloseHandle(gameProcess);
+    CloseHandle(exceptionMailSlot);
+
+    pBuf = nullptr;
+    hMapFile = INVALID_HANDLE_VALUE;
+    gameProcess = INVALID_HANDLE_VALUE;
+    exceptionMailSlot = INVALID_HANDLE_VALUE;
+}
+
+void GameInterface::detachDll() {
+    HMODULE dllModule = Utils::getModuleBaseAddress(gameProcess, RL_HOOK_DLL_NAME);
+    if (dllModule != nullptr)
+        Utils::detachDll(gameProcess, dllModule);
+}
+
+void GameInterface::checkIfFinished() const {
+    if (isFinished)
+        throw std::runtime_error("This object was already destructed, probably due to an exception!");
+}
+
+std::optional<py::array> GameInterface::step(bool readPixelBuffer) {
+    checkIfFinished();
+
+    checkForException();
+
+    try {
+        pipe.write(readPixelBuffer);
+        pipe.read<bool>();
+    } catch (...) {
+        checkForException();
+        throw;
+    }
+
+    if (!readPixelBuffer)
+        return std::nullopt;
+
+    return py::array(
+            pixelDataType == UBYTE ? py::dtype::of<std::uint8_t>() : py::dtype::of<std::float_t>(),
+            {height, width, channels},
+            {width * channels * dataTypeSize, channels * dataTypeSize, dataTypeSize},
+            pBuf
+    );
+}
+
+void GameInterface::setSpeed(std::double_t speed) {
+    checkIfFinished();
+    checkForException();
+    try {
+        timeDistorterPipe.write<std::uint8_t>(0);
+        timeDistorterPipe.write(speed);
+    } catch (...) {
+        checkForException();
+        throw;
+    }
+}
+
+void GameInterface::runAfap(std::double_t targetFramerate) {
+    checkIfFinished();
+    checkForException();
+    try {
+        timeDistorterPipe.write<std::uint8_t>(1);
+        timeDistorterPipe.write(targetFramerate);
+    } catch (...) {
+        checkForException();
+    }
+}
+
+void GameInterface::checkForException() {
+    DWORD cbMessage, cbRead;
+    BOOL result = GetMailslotInfo(exceptionMailSlot, nullptr, &cbMessage, nullptr, nullptr);
+    try {
+        Utils::checkError(!result, "GetMailslotInfo");
+    } catch (...) {
+        finish();
+        throw;
+    }
+    if (cbMessage != MAILSLOT_NO_MESSAGE) {
+        std::string message(cbMessage, '\0');
+        result = ReadFile(exceptionMailSlot, &message[0], cbMessage, &cbRead, nullptr);
+
+        try {
+            Utils::checkError(!result, "ReadFile");
+        } catch (...) {
+            finish();
+            throw;
+        }
+        finish();
+
+        if (cbRead != cbMessage)
+            throw std::runtime_error("Couldn't read exception mailslot");
+
+        throw std::runtime_error("Remote process failed at some point with the following error message: \"" + message + "\"");
+    }
+}
+
+std::uint32_t GameInterface::bufferSize() const {
+    return width * height * channels * dataTypeSize;
+}

RLHookLib/PyRLHook/src/GameInterface.h

@@ -0,0 +1,104 @@
+#pragma once
+#define NOMINMAX 
+#include <windows.h>
+#include <iostream>
+#include <pybind11/pybind11.h>
+#include <pybind11/numpy.h>
+#include <map>
+#include <atomic>
+#include <thread>
+#include <optional>
+#include <gl/GL.h>
+#include "src/Pipe.h"
+
+
+namespace py = pybind11;
+
+enum PixelFormat {
+    RED = GL_RED,
+    GREEN = GL_GREEN,
+    BLUE = GL_BLUE,
+    ALPHA = GL_ALPHA,
+    RGB = GL_RGB,
+    RGBA = GL_RGBA,
+};
+
+enum PixelDataType {
+    UBYTE = GL_UNSIGNED_BYTE,
+    FLOAT32 = GL_FLOAT
+};
+
+class GameInterface
+{
+
+public:
+	explicit GameInterface(DWORD pid, PixelFormat pixelFormat, PixelDataType pixelDataType);
+	explicit GameInterface(const std::string &processName, PixelFormat pixelFormat, PixelDataType pixelDataType);
+	~GameInterface();
+
+	std::optional<py::array> step(bool readPixelBuffer);
+	std::uint32_t bufferSize() const;
+
+    void setSpeed(std::double_t speed);
+    void runAfap(std::double_t targetFramerate);
+
+    void checkForException();
+
+	template <typename T> T read(const std::string &mod, const std::vector<DWORD> &offsets);
+	template <typename T> T read(LPCVOID adr);
+	template <typename T> void write(const std::string &mod, const std::vector<DWORD> &offsets, const T &value);
+	template <typename T> void write(LPVOID adr, const T &value);
+
+	static std::string basePath;
+
+private:
+
+    void detachDll();
+    void checkIfFinished() const;
+
+    void finish();
+
+	HANDLE gameProcess = INVALID_HANDLE_VALUE;
+	DWORD pid = 0;
+	Pipe pipe;
+	Pipe timeDistorterPipe;
+
+	std::map<std::string, LPVOID> moduleBases;
+
+    HANDLE exceptionMailSlot = INVALID_HANDLE_VALUE;
+
+    HANDLE hMapFile = INVALID_HANDLE_VALUE;
+    LPVOID pBuf = nullptr;
+	std::uint32_t width = 0;
+	std::uint32_t height = 0;
+    std::uint32_t channels = 0;
+    std::uint32_t dataTypeSize = 0;
+    PixelFormat pixelFormat;
+    PixelDataType pixelDataType;
+
+	bool isFinished = false;
+};
+
+template<typename T>
+T GameInterface::read(const std::string &mod, const std::vector<DWORD> &offsets) {
+    if (moduleBases.find(mod) == moduleBases.end())
+        moduleBases[mod] = Utils::getModuleBaseAddress(gameProcess, mod);
+    return Utils::template read<T>(gameProcess, moduleBases[mod], offsets);
+}
+
+template<typename T>
+T GameInterface::read(LPCVOID adr) {
+    return Utils::template read<T>(gameProcess, adr);
+}
+
+template<typename T>
+void GameInterface::write(const std::string &mod, const std::vector<DWORD> &offsets, const T &value) {
+    if (moduleBases.find(mod) == moduleBases.end())
+        moduleBases[mod] = Utils::getModuleBaseAddress(gameProcess, mod);
+    Utils::template write<T>(gameProcess, moduleBases[mod], offsets, value);
+}
+
+template<typename T>
+void GameInterface::write(LPVOID adr, const T & value) {
+    Utils::template write<T>(gameProcess, adr, value);
+}