[gui] GGUI 3/n: Add dependencies, interfaces, and backend-independent…

… code (taichi-dev#2650)

* backend independent code

* fix convo

* fix convo

* remove redundant additions to TAICHI_CORE_SOURCE

* add submodules ..

* fix convo

.gitmodules

@@ -31,3 +31,9 @@
 [submodule "external/VulkanMemoryAllocator"]
 	path = external/VulkanMemoryAllocator
 	url = https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator
+[submodule "external/imgui"]
+	path = external/imgui
+	url = https://github.com/ocornut/imgui.git
+[submodule "external/glm"]
+	path = external/glm
+	url = https://github.com/g-truc/glm.git

cmake/TaichiCore.cmake

@@ -36,11 +36,19 @@ if (WIN32)
     endif()
 endif()
 
+set(TI_WITH_GGUI OFF)
+if(TI_WITH_CUDA AND TI_WITH_VULKAN)
+    set(TI_WITH_GGUI ON)
+endif()
+
+
 if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/external/glad/src/glad.c")
     set(TI_WITH_OPENGL OFF)
     message(WARNING "external/glad submodule not detected. Settings TI_WITH_OPENGL to OFF.")
 endif()
 
+
+
 file(GLOB TAICHI_CORE_SOURCE
         "taichi/*/*/*/*.cpp" "taichi/*/*/*.cpp" "taichi/*/*.cpp" "taichi/*.cpp"
         "taichi/*/*/*/*.h" "taichi/*/*/*.h" "taichi/*/*.h" "taichi/*.h" "tests/cpp/task/*.cpp")
@@ -54,6 +62,30 @@ file(GLOB TAICHI_METAL_SOURCE "taichi/backends/metal/*.h" "taichi/backends/metal
 file(GLOB TAICHI_OPENGL_SOURCE "taichi/backends/opengl/*.h" "taichi/backends/opengl/*.cpp" "taichi/backends/opengl/shaders/*")
 file(GLOB TAICHI_CC_SOURCE "taichi/backends/cc/*.h" "taichi/backends/cc/*.cpp")
 file(GLOB TAICHI_VULKAN_SOURCE "taichi/backends/vulkan/*.h" "taichi/backends/vulkan/*.cpp" "taichi/backends/vulkan/shaders/*")
+
+file(GLOB TAICHI_GGUI_SOURCE
+    "taichi/ui/*.cpp"  "taichi/ui/*/*.cpp" "taichi/ui/*/*/*.cpp"  "taichi/ui/*/*/*/*.cpp" "taichi/ui/*/*/*/*/*.cpp"
+    "taichi/ui/*.h"  "taichi/ui/*/*.h" "taichi/ui/*/*/*.h"  "taichi/ui/*/*/*/*.h" "taichi/ui/*/*/*/*/*.h"
+    "taichi/ui/backends/vulkan/renderables/kernels.cu"
+)
+list(REMOVE_ITEM TAICHI_CORE_SOURCE ${TAICHI_GGUI_SOURCE})
+
+
+if(TI_WITH_GGUI)
+    add_definitions(-DTI_WITH_GGUI)
+
+    enable_language(CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -use_fast_math -std=c++17" )
+    list(APPEND TAICHI_CORE_SOURCE ${TAICHI_GGUI_SOURCE})
+
+    include_directories(SYSTEM external/glm)
+
+endif()
+
+
+
+
+
 # These are required, regardless of whether Vulkan is enabled or not
 # TODO(#2298): Clean up the Vulkan code structure, all Vulkan API related things should be
 # guarded by TI_WITH_VULKAN macro at the source code level.
@@ -295,3 +327,16 @@ if (WIN32)
     set_target_properties(${CORE_WITH_PYBIND_LIBRARY_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
             "${CMAKE_CURRENT_SOURCE_DIR}/runtimes")
 endif ()
+
+
+if(TI_WITH_GGUI)
+
+    # Dear ImGui
+    add_definitions(-DIMGUI_IMPL_VULKAN_NO_PROTOTYPES)
+    set(IMGUI_DIR external/imgui)
+    include_directories(external/glfw/include)
+    include_directories(SYSTEM ${IMGUI_DIR} ${IMGUI_DIR}/backends ..)
+    add_library(imgui  ${IMGUI_DIR}/backends/imgui_impl_glfw.cpp ${IMGUI_DIR}/backends/imgui_impl_vulkan.cpp ${IMGUI_DIR}/imgui.cpp ${IMGUI_DIR}/imgui_draw.cpp  ${IMGUI_DIR}/imgui_tables.cpp ${IMGUI_DIR}/imgui_widgets.cpp)
+    target_link_libraries(${CORE_LIBRARY_NAME} imgui)
+
+endif()

taichi/ui/common/app_config.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <string>
+#include "taichi/ui/utils/utils.h"
+#include "taichi/program/arch.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+struct AppConfig {
+  std::string name;
+  int width{0};
+  int height{0};
+  bool vsync{false};
+  std::string package_path;
+  taichi::lang::Arch ti_arch;
+};
+
+TI_UI_NAMESPACE_END

taichi/ui/common/camera.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "taichi/ui/utils/utils.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+enum class ProjectionMode : int { Perspective = 0, Orthogonal = 1 };
+
+struct Camera {
+  glm::vec3 position;
+  glm::vec3 lookat;
+  glm::vec3 up;
+  ProjectionMode projection_mode = ProjectionMode::Perspective;
+
+  float fov{45};
+
+  float left{-1};
+  float right{1};
+  float top{-1};
+  float bottom{1};
+  float z_near{0.1};
+  float z_far{1000};
+
+  glm::mat4 get_view_matrix() {
+    return glm::lookAt(position, lookat, up);
+  }
+  glm::mat4 get_projection_matrix(float aspect_ratio) {
+    if (projection_mode == ProjectionMode::Perspective) {
+      return glm::perspective(fov, aspect_ratio, z_far, z_near);
+    } else if (projection_mode == ProjectionMode::Orthogonal) {
+      return glm::ortho(left, right, top, bottom, z_far, z_near);
+    } else {
+      throw std::runtime_error("invalid camera projection mode");
+    }
+  }
+};
+
+TI_UI_NAMESPACE_END

taichi/ui/common/canvas_base.h

@@ -0,0 +1,41 @@
+#pragma once
+#include "taichi/ui/common/field_info.h"
+#include "taichi/ui/common/scene_base.h"
+#include "taichi/ui/common/renderable_info.h"
+#include "taichi/ui/utils/utils.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+struct SetImageInfo {
+  FieldInfo img;
+};
+
+struct TrianglesInfo {
+  RenderableInfo renderable_info;
+  glm::vec3 color;
+};
+
+struct CirclesInfo {
+  RenderableInfo renderable_info;
+  glm::vec3 color;
+  float radius;
+};
+
+struct LinesInfo {
+  RenderableInfo renderable_info;
+  glm::vec3 color;
+  float width;
+};
+
+class CanvasBase {
+ public:
+  virtual void set_background_color(const glm::vec3 &color) = 0;
+  virtual void set_image(const SetImageInfo &info) = 0;
+  virtual void triangles(const TrianglesInfo &info) = 0;
+  virtual void circles(const CirclesInfo &info) = 0;
+  virtual void lines(const LinesInfo &info) = 0;
+  virtual void scene(SceneBase *scene) = 0;
+  virtual ~CanvasBase() = default;
+};
+
+TI_UI_NAMESPACE_END

taichi/ui/common/event.h

@@ -0,0 +1,14 @@
+#pragma once
+#include "taichi/ui/utils/utils.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+enum class EventType : int { Any = 0, Press = 1, Release = 2 };
+
+struct Event {
+  EventType tag;
+
+  DEFINE_PROPERTY(std::string, key);
+};
+
+TI_UI_NAMESPACE_END

taichi/ui/common/field_info.h

@@ -0,0 +1,33 @@
+#pragma once
+#include "taichi/ui/utils/utils.h"
+
+#include "taichi/ir/type_utils.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+enum class FieldSource : int {
+  TaichiCuda = 0,
+  TaichiX64 = 1,
+  TaichiVulkan = 2,
+  TaichiOpenGL = 3
+  // support np array / torch tensor in the future?
+};
+
+enum class FieldType : int { Scalar = 0, Matrix = 1 };
+
+struct FieldInfo {
+  DEFINE_PROPERTY(bool, valid)
+  DEFINE_PROPERTY(FieldType, field_type);
+  DEFINE_PROPERTY(int, matrix_rows);
+  DEFINE_PROPERTY(int, matrix_cols);
+  DEFINE_PROPERTY(std::vector<int>, shape);
+  DEFINE_PROPERTY(FieldSource, field_source);
+  DEFINE_PROPERTY(taichi::lang::DataType, dtype);
+  DEFINE_PROPERTY(uint64_t, data);
+
+  FieldInfo() {
+    valid = false;
+  }
+};
+
+TI_UI_NAMESPACE_END

taichi/ui/common/gui_base.h

@@ -0,0 +1,26 @@
+#pragma once
+#include <string>
+#include "taichi/ui/utils/utils.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+class GuiBase {
+ public:
+  virtual void begin(std::string name,
+                     float x,
+                     float y,
+                     float width,
+                     float height) = 0;
+  virtual void end() = 0;
+  virtual void text(std::string text) = 0;
+  virtual bool checkbox(std::string name, bool old_value) = 0;
+  virtual float slider_float(std::string name,
+                             float old_value,
+                             float minimum,
+                             float maximum) = 0;
+  virtual glm::vec3 color_edit_3(std::string name, glm::vec3 old_value) = 0;
+  virtual bool button(std::string text) = 0;
+  virtual ~GuiBase() = default;
+};
+
+TI_UI_NAMESPACE_END

taichi/ui/common/input_handler.h

@@ -0,0 +1,104 @@
+#pragma once
+#include <memory>
+#include <functional>
+#include <vector>
+#include "taichi/ui/utils/utils.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+class InputHandler {
+ public:
+  void key_callback(GLFWwindow *window,
+                    int key,
+                    int scancode,
+                    int action,
+                    int mode) {
+    if (action == GLFW_PRESS) {
+      keys_[key] = true;
+    } else if (action == GLFW_RELEASE) {
+      keys_[key] = false;
+    }
+    for (auto f : user_key_callbacks_) {
+      f(key, action);
+    }
+  }
+
+  void mouse_pos_callback(GLFWwindow *window, double xpos, double ypos) {
+    if (first_mouse_) {
+      last_x_ = xpos;
+      last_y_ = ypos;
+      first_mouse_ = false;
+    }
+
+    last_x_ = xpos;
+    last_y_ = ypos;
+
+    for (auto f : user_mouse_pos_callbacks_) {
+      f(xpos, ypos);
+    }
+  }
+
+  void mouse_button_callback(GLFWwindow *window,
+                             int button,
+                             int action,
+                             int modifier) {
+    if (button == GLFW_MOUSE_BUTTON_LEFT) {
+      if (action == GLFW_PRESS) {
+        left_mouse_down_ = true;
+        glfwSetInputMode(window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN);
+      }
+      if (action == GLFW_RELEASE) {
+        left_mouse_down_ = false;
+        glfwSetInputMode(window, GLFW_CURSOR, GLFW_CURSOR_NORMAL);
+      }
+    }
+    if (action == GLFW_PRESS) {
+      keys_[button] = true;
+    } else if (action == GLFW_RELEASE) {
+      keys_[button] = false;
+    }
+    for (auto f : user_mouse_button_callbacks_) {
+      f(button, action);
+    }
+  }
+
+  bool is_pressed(int key) {
+    return keys_[key];
+  }
+
+  float last_x() {
+    return last_x_;
+  }
+
+  float last_y() {
+    return last_y_;
+  }
+
+  void add_key_callback(std::function<void(int, int)> f) {
+    user_key_callbacks_.push_back(f);
+  }
+  void add_mouse_pos_callback(std::function<void(double, double)> f) {
+    user_mouse_pos_callbacks_.push_back(f);
+  }
+  void add_mouse_button_callback(std::function<void(int, int)> f) {
+    user_mouse_button_callbacks_.push_back(f);
+  }
+
+  InputHandler() : keys_(1024, false) {
+  }
+
+ private:
+  bool first_mouse_ = true;
+
+  bool left_mouse_down_ = false;
+
+  std::vector<bool> keys_;
+  float last_x_ = 0;
+  float last_y_ = 0;
+
+  std::vector<std::function<void(int, int)>> user_key_callbacks_;
+  std::vector<std::function<void(double, double)>> user_mouse_pos_callbacks_;
+  std::vector<std::function<void(int, int)>> user_mouse_button_callbacks_;
+};
+
+TI_UI_NAMESPACE_END

taichi/ui/common/renderable_info.h

@@ -0,0 +1,15 @@
+#pragma once
+#include "taichi/ui/common/field_info.h"
+#include "taichi/ui/utils/utils.h"
+
+TI_UI_NAMESPACE_BEGIN
+
+struct RenderableInfo {
+  FieldInfo vertices;
+  FieldInfo normals;
+  FieldInfo tex_coords;
+  FieldInfo per_vertex_color;
+  FieldInfo indices;
+};
+
+TI_UI_NAMESPACE_END