Feature/job system

Axiom/CMakeLists.txt

@@ -351,6 +351,9 @@ function(axiom_configure_module_target target_name)
     AXIOM_SOURCE_DIR="${CMAKE_SOURCE_DIR}"
     AXIOM_ENABLE_WEBRTC=$<IF:$<BOOL:${AXIOM_ENABLE_WEBRTC}>,1,0>
     AXIOM_THREADED_RENDER=$<IF:$<BOOL:${AXIOM_THREADED_RENDER}>,1,0>
+    AXIOM_PARALLEL_CULL=$<IF:$<BOOL:${AXIOM_PARALLEL_CULL}>,1,0>
+    AXIOM_VERIFY_PARALLEL_CULL=$<IF:$<BOOL:${AXIOM_VERIFY_PARALLEL_CULL}>,1,0>
+    AXIOM_FRAME_TASK_GRAPH=$<IF:$<BOOL:${AXIOM_FRAME_TASK_GRAPH}>,1,0>
     AXIOM_WEBRTC_LINKED=${AXIOM_WEBRTC_LINKED}
   )
 
@@ -370,13 +373,7 @@ function(axiom_configure_module_target target_name)
     target_link_options(${target_name} PUBLIC ${AXIOM_WEBRTC_LINK_OPTIONS})
   endif()
 
-  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
-    target_compile_options(${target_name} PRIVATE
-      $<$<CONFIG:Release>:-O3 -march=native>
-      $<$<CONFIG:RelWithDebInfo>:-O2 -g -march=native>
-      $<$<CONFIG:MinSizeRel>:-Os>
-    )
-  endif()
+  axiom_apply_performance_options(${target_name})
 
   if(AXIOM_ENABLE_TSAN)
     if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
@@ -539,6 +536,9 @@ target_compile_definitions(AxiomRendererVulkanObjects PRIVATE
   AXIOM_SOURCE_DIR="${CMAKE_SOURCE_DIR}"
   AXIOM_ENABLE_WEBRTC=$<IF:$<BOOL:${AXIOM_ENABLE_WEBRTC}>,1,0>
   AXIOM_THREADED_RENDER=$<IF:$<BOOL:${AXIOM_THREADED_RENDER}>,1,0>
+  AXIOM_PARALLEL_CULL=$<IF:$<BOOL:${AXIOM_PARALLEL_CULL}>,1,0>
+  AXIOM_VERIFY_PARALLEL_CULL=$<IF:$<BOOL:${AXIOM_VERIFY_PARALLEL_CULL}>,1,0>
+  AXIOM_FRAME_TASK_GRAPH=$<IF:$<BOOL:${AXIOM_FRAME_TASK_GRAPH}>,1,0>
   AXIOM_WEBRTC_LINKED=${AXIOM_WEBRTC_LINKED}
 )
 if(AXIOM_ENABLE_TSAN)
@@ -547,6 +547,7 @@ if(AXIOM_ENABLE_TSAN)
     -fno-omit-frame-pointer
   )
 endif()
+axiom_apply_performance_options(AxiomRendererVulkanObjects)
 
 if(AXIOM_ENABLE_SCRIPTING)
   set(AXIOM_CORAL_MANAGED_DIR

Axiom/Core/Application.cpp

@@ -66,6 +66,8 @@ Application::Application(const ApplicationConfig &Config,
         .Width = m_Window->GetWidth(),
         .Height = m_Window->GetHeight(),
         .EnableThreadedRendering = m_Config.EnableThreadedRendering,
+        .EnableParallelCull = m_Config.EnableParallelCull,
+        .VerifyParallelCull = m_Config.VerifyParallelCull,
     });
   }
   Jobs::Startup();

Axiom/Core/Application.h

@@ -15,6 +15,15 @@ namespace Axiom {
 #ifndef AXIOM_THREADED_RENDER
 #define AXIOM_THREADED_RENDER 0
 #endif
+#ifndef AXIOM_PARALLEL_CULL
+#define AXIOM_PARALLEL_CULL 0
+#endif
+#ifndef AXIOM_VERIFY_PARALLEL_CULL
+#define AXIOM_VERIFY_PARALLEL_CULL 0
+#endif
+#ifndef AXIOM_FRAME_TASK_GRAPH
+#define AXIOM_FRAME_TASK_GRAPH 0
+#endif
 
 class Renderer;
 struct RendererDeleter {
@@ -40,6 +49,9 @@ struct ApplicationConfig {
   RuntimeMode Mode{RuntimeMode::LocalWindowedEditor};
   IViewportFrameOutput *FrameOutput{nullptr};
   bool EnableThreadedRendering{AXIOM_THREADED_RENDER != 0};
+  bool EnableParallelCull{AXIOM_PARALLEL_CULL != 0};
+  bool VerifyParallelCull{AXIOM_VERIFY_PARALLEL_CULL != 0};
+  bool EnableFrameTaskGraph{AXIOM_FRAME_TASK_GRAPH != 0};
 };
 
 class Application {
@@ -65,6 +77,9 @@ class Application {
   [[nodiscard]] float GetDeltaTime() const { return m_DeltaTime; }
   [[nodiscard]] uint64_t GetFrameIndex() const { return m_FrameIndex; }
   [[nodiscard]] RuntimeMode GetRuntimeMode() const { return m_Config.Mode; }
+  [[nodiscard]] bool IsFrameTaskGraphEnabled() const {
+    return m_Config.EnableFrameTaskGraph;
+  }
   [[nodiscard]] Renderer &GetRenderer() const;
   [[nodiscard]] Renderer *TryGetRenderer() const;
   [[nodiscard]] ModuleManager &GetModuleManager() { return m_ModuleManager; }

Axiom/Jobs/JobSystem.cpp

@@ -3,31 +3,48 @@
 #include "Core/Threading.h"
 #include "Jobs/TaskScheduler.h"
 
+#include <array>
 #include <algorithm>
+#include <atomic>
+#include <cstdint>
 #include <memory>
 #include <mutex>
 #include <string>
 #include <utility>
 #include <vector>
 
 namespace Axiom::Jobs {
-struct JobState {
-  std::unique_ptr<enki::ITaskSet> Task;
-  std::vector<JobHandle> DependencyHandles;
-};
-
 namespace {
+constexpr size_t kJobPoolCapacity = 131072;
+constexpr size_t kMaxInlineDependencies = 16;
+
 void OnWorkerThreadStart(uint32_t ThreadNum);
 
 class LambdaTaskSet final : public enki::ITaskSet {
 public:
-  explicit LambdaTaskSet(JobFn Function)
-      : enki::ITaskSet(1), m_Function(std::move(Function)) {}
+  LambdaTaskSet() : enki::ITaskSet(1) {}
+
+  void Reset(JobFn Function, std::span<const JobHandle> Dependencies) {
+    m_SetSize = 1;
+    m_MinRange = 1;
+    m_Function = std::move(Function);
+    m_DependencyCount = std::min(Dependencies.size(), m_Dependencies.size());
+    for (size_t Index = 0; Index < m_DependencyCount; ++Index) {
+      m_Dependencies[Index] = Dependencies[Index];
+    }
+  }
 
-  void ExecuteRange(enki::TaskSetPartition, uint32_t) override { m_Function(); }
+  void Clear() {
+    m_Function.Reset();
+    m_DependencyCount = 0;
+  }
+
+  void ExecuteRange(enki::TaskSetPartition, uint32_t) override;
 
 private:
   JobFn m_Function;
+  std::array<JobHandle, kMaxInlineDependencies> m_Dependencies;
+  size_t m_DependencyCount{0};
 };
 
 class ParallelForTaskSet final : public enki::ITaskSet {
@@ -51,6 +68,16 @@ class ParallelForTaskSet final : public enki::ITaskSet {
   ParallelForFn m_Function;
 };
 
+} // namespace
+
+struct JobState {
+  LambdaTaskSet Task;
+  std::atomic<uint32_t> Generation{0};
+  std::atomic<bool> Recycled{true};
+};
+
+namespace {
+
 class JobSystem {
 public:
   void Startup() {
@@ -60,7 +87,9 @@ class JobSystem {
       m_Scheduler = std::make_unique<enki::TaskScheduler>();
       enki::TaskSchedulerConfig Config = m_Scheduler->GetConfig();
       Config.profilerCallbacks.threadStart = &OnWorkerThreadStart;
+      Config.numExternalTaskThreads = 4;
       m_Scheduler->Initialize(Config);
+      ResetPool();
     }
   }
 
@@ -74,54 +103,77 @@ class JobSystem {
     if (m_StartupCount == 0 && m_Scheduler != nullptr) {
       m_Scheduler->WaitforAllAndShutdown();
       m_Scheduler.reset();
+      ResetPool();
     }
   }
 
   JobHandle ScheduleJob(JobFn Function) {
-    auto State = std::make_shared<JobState>();
-    State->Task = std::make_unique<LambdaTaskSet>(std::move(Function));
-    m_Scheduler->AddTaskSetToPipe(State->Task.get());
-    return {.State = std::move(State)};
+    if (!CanUseScheduler()) {
+      Function();
+      return {};
+    }
+
+    JobHandle Handle = AcquireTask(Function, {});
+    if (!Handle.IsValid()) {
+      Function();
+      return {};
+    }
+
+    m_Scheduler->AddTaskSetToPipe(&Handle.State->Task);
+    return Handle;
   }
 
   JobHandle ScheduleJobAfter(JobFn Function, std::span<JobHandle> Deps) {
-    auto State = std::make_shared<JobState>();
-    State->DependencyHandles.reserve(Deps.size());
-    for (const JobHandle &Dependency : Deps) {
-      if (!Dependency.IsValid() || Dependency.State->Task == nullptr) {
-        continue;
+    if (!CanUseScheduler()) {
+      for (const JobHandle &Dependency : Deps) {
+        Wait(Dependency);
       }
-      State->DependencyHandles.push_back(Dependency);
+      Function();
+      return {};
     }
 
-    State->Task = std::make_unique<LambdaTaskSet>(
-        [this, State, Function = std::move(Function)]() mutable {
-          for (const JobHandle &Dependency : State->DependencyHandles) {
-            if (!Dependency.IsValid() || Dependency.State->Task == nullptr) {
-              continue;
-            }
+    if (Deps.size() > kMaxInlineDependencies) {
+      for (size_t Index = kMaxInlineDependencies; Index < Deps.size();
+           ++Index) {
+        Wait(Deps[Index]);
+      }
+    }
 
-            m_Scheduler->WaitforTask(Dependency.State->Task.get());
-          }
+    const size_t InlineDependencyCount =
+        std::min(Deps.size(), kMaxInlineDependencies);
+    JobHandle Handle =
+        AcquireTask(Function, Deps.first(InlineDependencyCount));
+    if (!Handle.IsValid()) {
+      for (size_t Index = 0; Index < InlineDependencyCount; ++Index) {
+        Wait(Deps[Index]);
+      }
+      Function();
+      return {};
+    }
 
-          Function();
-        });
-    m_Scheduler->AddTaskSetToPipe(State->Task.get());
-    return {.State = std::move(State)};
+    m_Scheduler->AddTaskSetToPipe(&Handle.State->Task);
+    return Handle;
   }
 
   void Wait(JobHandle Handle) {
-    if (!Handle.IsValid() || Handle.State->Task == nullptr) {
+    if (m_Scheduler == nullptr || !IsCurrent(Handle)) {
       return;
     }
 
-    m_Scheduler->WaitforTask(Handle.State->Task.get());
+    m_Scheduler->WaitforTask(&Handle.State->Task);
+    ReleaseTask(Handle);
   }
 
   void ParallelFor(size_t Count, ParallelForFn Function) {
     if (Count == 0) {
       return;
     }
+    if (!CanUseScheduler()) {
+      for (size_t Index = 0; Index < Count; ++Index) {
+        Function(Index);
+      }
+      return;
+    }
 
     ParallelForTaskSet Task(Count, std::move(Function));
     m_Scheduler->AddTaskSetToPipe(&Task);
@@ -131,7 +183,72 @@ class JobSystem {
 private:
   std::mutex m_Mutex;
   std::unique_ptr<enki::TaskScheduler> m_Scheduler;
+  std::unique_ptr<JobState[]> m_TaskPool;
+  std::vector<JobState *> m_FreeList;
   size_t m_StartupCount{0};
+
+  void ResetPool() {
+    if (m_TaskPool == nullptr) {
+      m_TaskPool = std::make_unique<JobState[]>(kJobPoolCapacity);
+    }
+
+    m_FreeList.reserve(kJobPoolCapacity);
+    m_FreeList.clear();
+    for (size_t Index = 0; Index < kJobPoolCapacity; ++Index) {
+      JobState &State = m_TaskPool[Index];
+      State.Task.Clear();
+      State.Generation.fetch_add(1, std::memory_order_relaxed);
+      State.Recycled.store(true, std::memory_order_relaxed);
+      m_FreeList.push_back(&State);
+    }
+  }
+
+  JobHandle AcquireTask(JobFn &Function, std::span<const JobHandle> Deps) {
+    std::scoped_lock Lock(m_Mutex);
+    if (m_TaskPool == nullptr || m_FreeList.empty()) {
+      return {};
+    }
+
+    JobState *State = m_FreeList.back();
+    m_FreeList.pop_back();
+    State->Recycled.store(false, std::memory_order_relaxed);
+    const uint32_t Generation =
+        State->Generation.fetch_add(1, std::memory_order_relaxed) + 1;
+    State->Task.Reset(std::move(Function), Deps);
+    return {.State = State, .Generation = Generation};
+  }
+
+  void ReleaseTask(JobHandle Handle) {
+    JobState *State = Handle.State;
+    if (State == nullptr ||
+        State->Generation.load(std::memory_order_acquire) !=
+            Handle.Generation ||
+        State->Recycled.exchange(true, std::memory_order_acq_rel)) {
+      return;
+    }
+
+    State->Task.Clear();
+
+    std::scoped_lock Lock(m_Mutex);
+    m_FreeList.push_back(State);
+  }
+
+  bool IsCurrent(JobHandle Handle) const {
+    return Handle.State != nullptr &&
+           Handle.State->Generation.load(std::memory_order_acquire) ==
+               Handle.Generation &&
+           !Handle.State->Recycled.load(std::memory_order_acquire);
+  }
+
+  bool CanUseScheduler() {
+    if (m_Scheduler == nullptr) {
+      return false;
+    }
+    if (m_Scheduler->GetThreadNum() != enki::NO_THREAD_NUM) {
+      return true;
+    }
+    return m_Scheduler->RegisterExternalTaskThread();
+  }
 };
 
 JobSystem &GetJobSystem() {
@@ -143,6 +260,14 @@ void OnWorkerThreadStart(uint32_t ThreadNum) {
   Threading::SetCurrentThreadName("Axiom Job Worker " +
                                   std::to_string(ThreadNum));
 }
+
+void LambdaTaskSet::ExecuteRange(enki::TaskSetPartition, uint32_t) {
+  for (size_t Index = 0; Index < m_DependencyCount; ++Index) {
+    GetJobSystem().Wait(m_Dependencies[Index]);
+  }
+
+  m_Function();
+}
 } // namespace
 
 void Startup() { GetJobSystem().Startup(); }