From be07e6aa6f040fbba64235393c0bfbb224386651 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 18:04:23 +0300 Subject: [PATCH 01/30] implement coroutine allocator --- .../allocator/coroutine-allocator.cpp | 93 +++++++++++++++++++ runtime-light/allocator/coroutine-allocator.h | 34 +++++++ 2 files changed, 127 insertions(+) create mode 100644 runtime-light/allocator/coroutine-allocator.cpp create mode 100644 runtime-light/allocator/coroutine-allocator.h diff --git a/runtime-light/allocator/coroutine-allocator.cpp b/runtime-light/allocator/coroutine-allocator.cpp new file mode 100644 index 0000000000..bd094a5380 --- /dev/null +++ b/runtime-light/allocator/coroutine-allocator.cpp @@ -0,0 +1,93 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include +#include + +#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/k2-platform/k2-api.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +CoroutineAllocator::CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) + : m_min_extra_mem_size(min_extra_mem_size) { + // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, + // oom_handling_mem_size); + void* buffer{RuntimeAllocator::get().alloc_global_memory(mem_size)}; + memory_resource.init(buffer, mem_size, oom_handling_mem_size); +} + +auto CoroutineAllocator::init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) -> void { + kphp::log::assertion(buffer != nullptr); + // kphp::log::debug("init coroutine allocator -> {:p}: buffer -> {:p}, memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, + // mem_size, oom_handling_mem_size); + memory_resource.init(buffer, mem_size, oom_handling_mem_size); +} + +auto CoroutineAllocator::free() -> void { + // kphp::log::debug("free coroutine allocator -> {:p}", reinterpret_cast(this)); + auto* extra_memory{memory_resource.get_extra_memory_head()}; + while (extra_memory->get_pool_payload_size() != 0) { + auto* extra_memory_to_release{extra_memory}; + extra_memory = extra_memory->next_in_chain; + k2::free(extra_memory_to_release); + } + k2::free(memory_resource.memory_begin()); +} + +auto CoroutineAllocator::alloc_memory(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + void* mem{memory_resource.allocate(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = memory_resource.allocate(size); + kphp::log::assertion(mem != nullptr); + } + return mem; +} + +auto CoroutineAllocator::alloc0_memory(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + void* mem{memory_resource.allocate0(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = memory_resource.allocate0(size); + kphp::log::assertion(mem != nullptr); + } + return mem; +} + +auto CoroutineAllocator::realloc_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { + kphp::log::assertion(new_size > old_size); + void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; + if (new_mem == nullptr) [[unlikely]] { + request_extra_memory(new_size * 2); + new_mem = memory_resource.reallocate(old_mem, new_size, old_size); + kphp::log::assertion(new_mem != nullptr); + } + return new_mem; +} + +auto CoroutineAllocator::free_memory(void* mem, size_t size) noexcept -> void { + kphp::log::assertion(size != 0); + memory_resource.deallocate(mem, size); +} + +auto CoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> void { + // Extra mem size have to be greater than max chunk block + const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; + + size_t extra_mem_size{std::max(min_size, requested_size)}; + // Take into account internal layout of `memory_resource::extra_memory_pool` + extra_mem_size += sizeof(memory_resource::extra_memory_pool); + // The smallest power of two that is not smaller than `extra_mem_size` + extra_mem_size = std::bit_ceil(extra_mem_size); + + // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); + + auto* extra_mem{RuntimeAllocator::get().alloc_global_memory(extra_mem_size)}; + memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); +} diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h new file mode 100644 index 0000000000..30f1f433a2 --- /dev/null +++ b/runtime-light/allocator/coroutine-allocator.h @@ -0,0 +1,34 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" + +struct CoroutineAllocator final : vk::not_copyable { + static auto get() noexcept -> CoroutineAllocator&; + + CoroutineAllocator() = default; + CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size); + + void init(void* buffer, size_t mem_size, size_t oom_handling_mem_size); + void free(); + + auto alloc_memory(size_t size) noexcept -> void*; + auto alloc0_memory(size_t size) noexcept -> void*; + auto realloc_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_memory(void* mem, size_t size) noexcept -> void; + +private: + auto request_extra_memory(size_t requested_size) noexcept -> void; + +public: + memory_resource::unsynchronized_pool_resource memory_resource; + +private: + size_t m_min_extra_mem_size{0}; +}; From e0b50021bfe21f20a6cc82e575961b43979066c4 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 18:05:36 +0300 Subject: [PATCH 02/30] apply code style --- runtime-light/allocator/coroutine-allocator.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h index 30f1f433a2..3febba02b4 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/coroutine-allocator.h @@ -1,6 +1,6 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2026 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once From 0189223f56428e5457e4803ce6d7031ca6cc5846 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 18:09:23 +0300 Subject: [PATCH 03/30] move coroutine-allocator into kphp::coro namespace --- runtime-light/allocator/coroutine-allocator.cpp | 4 ++++ runtime-light/allocator/coroutine-allocator.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/runtime-light/allocator/coroutine-allocator.cpp b/runtime-light/allocator/coroutine-allocator.cpp index bd094a5380..1bebdcb6c5 100644 --- a/runtime-light/allocator/coroutine-allocator.cpp +++ b/runtime-light/allocator/coroutine-allocator.cpp @@ -12,6 +12,8 @@ #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" +namespace kphp::coro { + CoroutineAllocator::CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, @@ -91,3 +93,5 @@ auto CoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> auto* extra_mem{RuntimeAllocator::get().alloc_global_memory(extra_mem_size)}; memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); } + +} // namespace kphp::coro diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h index 3febba02b4..92888a4dd0 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/coroutine-allocator.h @@ -9,6 +9,8 @@ #include "common/mixin/not_copyable.h" #include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" +namespace kphp::coro { + struct CoroutineAllocator final : vk::not_copyable { static auto get() noexcept -> CoroutineAllocator&; @@ -32,3 +34,5 @@ struct CoroutineAllocator final : vk::not_copyable { private: size_t m_min_extra_mem_size{0}; }; + +} // namespace kphp::coro From 8825f2c019a48890857ee628fd5c06a36ed7dbe5 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 18:13:05 +0300 Subject: [PATCH 04/30] add noexcept to methods in coroutine-allocator --- runtime-light/allocator/coroutine-allocator.cpp | 6 +++--- runtime-light/allocator/coroutine-allocator.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/runtime-light/allocator/coroutine-allocator.cpp b/runtime-light/allocator/coroutine-allocator.cpp index 1bebdcb6c5..4a031b4074 100644 --- a/runtime-light/allocator/coroutine-allocator.cpp +++ b/runtime-light/allocator/coroutine-allocator.cpp @@ -14,7 +14,7 @@ namespace kphp::coro { -CoroutineAllocator::CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) +CoroutineAllocator::CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, // oom_handling_mem_size); @@ -22,14 +22,14 @@ CoroutineAllocator::CoroutineAllocator(size_t mem_size, size_t min_extra_mem_siz memory_resource.init(buffer, mem_size, oom_handling_mem_size); } -auto CoroutineAllocator::init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) -> void { +auto CoroutineAllocator::init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept -> void { kphp::log::assertion(buffer != nullptr); // kphp::log::debug("init coroutine allocator -> {:p}: buffer -> {:p}, memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, // mem_size, oom_handling_mem_size); memory_resource.init(buffer, mem_size, oom_handling_mem_size); } -auto CoroutineAllocator::free() -> void { +auto CoroutineAllocator::free() noexcept -> void { // kphp::log::debug("free coroutine allocator -> {:p}", reinterpret_cast(this)); auto* extra_memory{memory_resource.get_extra_memory_head()}; while (extra_memory->get_pool_payload_size() != 0) { diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h index 92888a4dd0..0d6ae39a93 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/coroutine-allocator.h @@ -14,11 +14,11 @@ namespace kphp::coro { struct CoroutineAllocator final : vk::not_copyable { static auto get() noexcept -> CoroutineAllocator&; - CoroutineAllocator() = default; - CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size); + CoroutineAllocator() noexcept = default; + CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; - void init(void* buffer, size_t mem_size, size_t oom_handling_mem_size); - void free(); + void init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept; + void free() noexcept; auto alloc_memory(size_t size) noexcept -> void*; auto alloc0_memory(size_t size) noexcept -> void*; From 2f7eccb2e54349b50d552df2d24582c322c673c0 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 18:39:36 +0300 Subject: [PATCH 05/30] add coroutine-allocator to cmake --- runtime-light/allocator/allocator.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake index 85880d002e..6f7fa53950 100644 --- a/runtime-light/allocator/allocator.cmake +++ b/runtime-light/allocator/allocator.cmake @@ -1 +1 @@ -set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp) +set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp allocator/coroutine-allocator.cpp) From 2eaaf2ae0a10a7b1f7ae131310001cdd0fc83ab6 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 18:42:39 +0300 Subject: [PATCH 06/30] add coroutine malloc interface --- .../core/allocator/detail/control-block.h | 49 +++++++ .../core/allocator/script-malloc-interface.h | 69 +++------- .../allocator/coroutine-malloc-interface.h | 120 ++++++++++++++++++ 3 files changed, 184 insertions(+), 54 deletions(-) create mode 100644 runtime-common/core/allocator/detail/control-block.h create mode 100644 runtime-light/allocator/coroutine-malloc-interface.h diff --git a/runtime-common/core/allocator/detail/control-block.h b/runtime-common/core/allocator/detail/control-block.h new file mode 100644 index 0000000000..a496b302cf --- /dev/null +++ b/runtime-common/core/allocator/detail/control-block.h @@ -0,0 +1,49 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include + +namespace kphp::memory::detail { + +struct control_block { +private: + static constexpr auto SIZE_FIELD_BITSIZE{48}; + static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; + static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; + static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; + + static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); + +public: + static constexpr uint64_t max_size() noexcept { + return 1UL << SIZE_FIELD_BITSIZE; + } + + static constexpr uint64_t max_alignment() noexcept { + return 1UL << BASE_OFFSET_FIELD_BITSIZE; + } + + uint64_t raw() const noexcept { + return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); + } + + static control_block from_raw(uint64_t raw) noexcept { + return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; + } + + uint64_t size : SIZE_FIELD_BITSIZE; + uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; +}; + +inline bool is_power_of_2(uint64_t v) noexcept { + return v && !(v & (v - 1)); +} + +static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); + +} // namespace kphp::memory::detail diff --git a/runtime-common/core/allocator/script-malloc-interface.h b/runtime-common/core/allocator/script-malloc-interface.h index 6af4350f87..638c01be8a 100644 --- a/runtime-common/core/allocator/script-malloc-interface.h +++ b/runtime-common/core/allocator/script-malloc-interface.h @@ -8,9 +8,9 @@ #include #include #include -#include #include "common/wrappers/likely.h" +#include "runtime-common/core/allocator/detail/control-block.h" #include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-common/core/utils/kphp-assert-core.h" @@ -22,48 +22,9 @@ namespace script { constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB -namespace details { -struct control_block { -private: - static constexpr auto SIZE_FIELD_BITSIZE{48}; - static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; - static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; - static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; - - static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); - -public: - static constexpr uint64_t max_size() noexcept { - return 1UL << SIZE_FIELD_BITSIZE; - } - - static constexpr uint64_t max_alignment() noexcept { - return 1UL << BASE_OFFSET_FIELD_BITSIZE; - } - - uint64_t raw() const noexcept { - return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); - } - - static control_block from_raw(uint64_t raw) noexcept { - return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; - } - - uint64_t size : SIZE_FIELD_BITSIZE; - uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; -}; - -inline bool is_power_of_2(uint64_t v) noexcept { - return v && !(v & (v - 1)); -} - -static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); - -} // namespace details - inline void* alloc(size_t size) noexcept { - constexpr size_t cb_size{sizeof(details::control_block)}; - if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; + if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); return nullptr; } @@ -73,22 +34,22 @@ inline void* alloc(size_t size) noexcept { php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); return base; } - *(static_cast(base)) = details::control_block{.size = total_size, .base_offset = cb_size}.raw(); + *(static_cast(base)) = kphp::memory::detail::control_block{.size = total_size, .base_offset = cb_size}.raw(); return static_cast(static_cast(base) + cb_size); } inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { // Check that provided alignment is power of two const size_t align{static_cast(alignment)}; - if (unlikely(align == 0 || !details::is_power_of_2(align) || align >= details::control_block::max_alignment())) { - php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", details::control_block::max_alignment(), - align); + if (unlikely(align == 0 || !kphp::memory::detail::is_power_of_2(align) || align >= kphp::memory::detail::control_block::max_alignment())) { + php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", + kphp::memory::detail::control_block::max_alignment(), align); return nullptr; } // Check that memory is enough - constexpr size_t cb_size{sizeof(details::control_block)}; - if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; + if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); return nullptr; } @@ -108,7 +69,7 @@ inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { // Save control block *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT - details::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); + kphp::memory::detail::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); return reinterpret_cast(aligned_u); // NOLINT } @@ -126,11 +87,11 @@ inline void free(void* ptr) noexcept { return; } - constexpr size_t cb_size{sizeof(details::control_block)}; + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; const auto mem{reinterpret_cast(ptr)}; - const auto cb{details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT RuntimeAllocator::get().free_script_memory(base, cb.size); } @@ -145,10 +106,10 @@ inline void* realloc(void* ptr, size_t new_size) noexcept { return nullptr; } - constexpr size_t cb_size{sizeof(details::control_block)}; + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; const auto mem{reinterpret_cast(ptr)}; - const auto cb{details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT const size_t old_size{cb.size}; diff --git a/runtime-light/allocator/coroutine-malloc-interface.h b/runtime-light/allocator/coroutine-malloc-interface.h new file mode 100644 index 0000000000..f4cdbc9842 --- /dev/null +++ b/runtime-light/allocator/coroutine-malloc-interface.h @@ -0,0 +1,120 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include + +#include "common/wrappers/likely.h" +#include "runtime-common/core/allocator/detail/control-block.h" +#include "runtime-common/core/utils/kphp-assert-core.h" +#include "runtime-light/allocator/coroutine-allocator.h" + +namespace kphp::memory::coro { + +constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB + +inline void* alloc(size_t size) noexcept { + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; + if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + const size_t total_size{size + cb_size}; + void* base{kphp::coro::CoroutineAllocator::get().alloc_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough coroutine memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + *(static_cast(base)) = kphp::memory::detail::control_block{.size = total_size, .base_offset = cb_size}.raw(); + return static_cast(static_cast(base) + cb_size); +} + +inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { + // Check that provided alignment is power of two + const size_t align{static_cast(alignment)}; + if (unlikely(align == 0 || !kphp::memory::detail::is_power_of_2(align) || align >= kphp::memory::detail::control_block::max_alignment())) { + php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", + kphp::memory::detail::control_block::max_alignment(), align); + return nullptr; + } + + // Check that memory is enough + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; + if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + + // Request mem from underlying memory manager + const size_t total_size{size + (align - 1) + cb_size}; + void* base{kphp::coro::CoroutineAllocator::get().alloc_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough coroutine memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + + const uint64_t base_u{reinterpret_cast(base)}; + // The smallest multiple of `align` greater than or equal to requested memory + const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; + const uint64_t base_offset_u{aligned_u - base_u}; + + // Save control block + *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT + kphp::memory::detail::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); + + return reinterpret_cast(aligned_u); // NOLINT +} + +inline void* calloc(size_t num, size_t size) noexcept { + void* ptr{kphp::memory::coro::alloc(num * size)}; + if (unlikely(ptr == nullptr)) { + return nullptr; + } + return std::memset(ptr, 0, num * size); +} + +inline void free(void* ptr) noexcept { + if (unlikely(ptr == nullptr)) { + return; + } + + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + + kphp::coro::CoroutineAllocator::get().free_memory(base, cb.size); +} + +inline void* realloc(void* ptr, size_t new_size) noexcept { + if (unlikely(ptr == nullptr)) { + return kphp::memory::coro::alloc(new_size); + } + + if (unlikely(new_size == 0)) { + kphp::memory::coro::free(ptr); + return nullptr; + } + + constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + + void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + const size_t old_size{cb.size}; + + void* new_ptr{kphp::memory::coro::alloc(new_size)}; + if (likely(new_ptr != nullptr)) { + std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); + kphp::coro::CoroutineAllocator::get().free_memory(old_base, old_size); + } + return new_ptr; +} + +} // namespace kphp::memory::coro From f4ca83ca02304d8ef403ffcaf3205465473e1ea6 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 18:43:26 +0300 Subject: [PATCH 07/30] remove excess include in control-block --- runtime-common/core/allocator/detail/control-block.h | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime-common/core/allocator/detail/control-block.h b/runtime-common/core/allocator/detail/control-block.h index a496b302cf..d41bb6040c 100644 --- a/runtime-common/core/allocator/detail/control-block.h +++ b/runtime-common/core/allocator/detail/control-block.h @@ -4,7 +4,6 @@ #pragma once -#include #include #include From 1a3b5858723785ba57528b7a019cbf29d663e33f Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 19:09:33 +0300 Subject: [PATCH 08/30] rename coroutine-allocator to runtime-coroutine-allocator --- runtime-light/allocator/allocator.cmake | 2 +- .../allocator/coroutine-malloc-interface.h | 10 ++++----- ...or.cpp => runtime-coroutine-allocator.cpp} | 22 ++++++++----------- ...ocator.h => runtime-coroutine-allocator.h} | 12 ++++------ 4 files changed, 19 insertions(+), 27 deletions(-) rename runtime-light/allocator/{coroutine-allocator.cpp => runtime-coroutine-allocator.cpp} (79%) rename runtime-light/allocator/{coroutine-allocator.h => runtime-coroutine-allocator.h} (73%) diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake index 6f7fa53950..777c6117b6 100644 --- a/runtime-light/allocator/allocator.cmake +++ b/runtime-light/allocator/allocator.cmake @@ -1 +1 @@ -set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp allocator/coroutine-allocator.cpp) +set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp allocator/runtime-coroutine-allocator.cpp) diff --git a/runtime-light/allocator/coroutine-malloc-interface.h b/runtime-light/allocator/coroutine-malloc-interface.h index f4cdbc9842..e05faaa5b4 100644 --- a/runtime-light/allocator/coroutine-malloc-interface.h +++ b/runtime-light/allocator/coroutine-malloc-interface.h @@ -11,7 +11,7 @@ #include "common/wrappers/likely.h" #include "runtime-common/core/allocator/detail/control-block.h" #include "runtime-common/core/utils/kphp-assert-core.h" -#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" namespace kphp::memory::coro { @@ -24,7 +24,7 @@ inline void* alloc(size_t size) noexcept { return nullptr; } const size_t total_size{size + cb_size}; - void* base{kphp::coro::CoroutineAllocator::get().alloc_memory(total_size)}; + void* base{RuntimeCoroutineAllocator::get().alloc_memory(total_size)}; if (unlikely(base == nullptr)) { php_warning("not enough coroutine memory to allocate, requested : %lu, actual requested: %lu", size, total_size); return base; @@ -51,7 +51,7 @@ inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { // Request mem from underlying memory manager const size_t total_size{size + (align - 1) + cb_size}; - void* base{kphp::coro::CoroutineAllocator::get().alloc_memory(total_size)}; + void* base{RuntimeCoroutineAllocator::get().alloc_memory(total_size)}; if (unlikely(base == nullptr)) { php_warning("not enough coroutine memory to allocate, requested : %lu, actual requested: %lu", size, total_size); return base; @@ -88,7 +88,7 @@ inline void free(void* ptr) noexcept { const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - kphp::coro::CoroutineAllocator::get().free_memory(base, cb.size); + RuntimeCoroutineAllocator::get().free_memory(base, cb.size); } inline void* realloc(void* ptr, size_t new_size) noexcept { @@ -112,7 +112,7 @@ inline void* realloc(void* ptr, size_t new_size) noexcept { void* new_ptr{kphp::memory::coro::alloc(new_size)}; if (likely(new_ptr != nullptr)) { std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - kphp::coro::CoroutineAllocator::get().free_memory(old_base, old_size); + RuntimeCoroutineAllocator::get().free_memory(old_base, old_size); } return new_ptr; } diff --git a/runtime-light/allocator/coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp similarity index 79% rename from runtime-light/allocator/coroutine-allocator.cpp rename to runtime-light/allocator/runtime-coroutine-allocator.cpp index 4a031b4074..5d0eeb36ce 100644 --- a/runtime-light/allocator/coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -8,13 +8,11 @@ #include #include "runtime-common/core/allocator/runtime-allocator.h" -#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" -namespace kphp::coro { - -CoroutineAllocator::CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept +RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, // oom_handling_mem_size); @@ -22,14 +20,14 @@ CoroutineAllocator::CoroutineAllocator(size_t mem_size, size_t min_extra_mem_siz memory_resource.init(buffer, mem_size, oom_handling_mem_size); } -auto CoroutineAllocator::init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept -> void { +auto RuntimeCoroutineAllocator::init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept -> void { kphp::log::assertion(buffer != nullptr); // kphp::log::debug("init coroutine allocator -> {:p}: buffer -> {:p}, memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, // mem_size, oom_handling_mem_size); memory_resource.init(buffer, mem_size, oom_handling_mem_size); } -auto CoroutineAllocator::free() noexcept -> void { +auto RuntimeCoroutineAllocator::free() noexcept -> void { // kphp::log::debug("free coroutine allocator -> {:p}", reinterpret_cast(this)); auto* extra_memory{memory_resource.get_extra_memory_head()}; while (extra_memory->get_pool_payload_size() != 0) { @@ -40,7 +38,7 @@ auto CoroutineAllocator::free() noexcept -> void { k2::free(memory_resource.memory_begin()); } -auto CoroutineAllocator::alloc_memory(size_t size) noexcept -> void* { +auto RuntimeCoroutineAllocator::alloc_memory(size_t size) noexcept -> void* { kphp::log::assertion(size != 0); void* mem{memory_resource.allocate(size)}; if (mem == nullptr) [[unlikely]] { @@ -51,7 +49,7 @@ auto CoroutineAllocator::alloc_memory(size_t size) noexcept -> void* { return mem; } -auto CoroutineAllocator::alloc0_memory(size_t size) noexcept -> void* { +auto RuntimeCoroutineAllocator::alloc0_memory(size_t size) noexcept -> void* { kphp::log::assertion(size != 0); void* mem{memory_resource.allocate0(size)}; if (mem == nullptr) [[unlikely]] { @@ -62,7 +60,7 @@ auto CoroutineAllocator::alloc0_memory(size_t size) noexcept -> void* { return mem; } -auto CoroutineAllocator::realloc_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { +auto RuntimeCoroutineAllocator::realloc_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { kphp::log::assertion(new_size > old_size); void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; if (new_mem == nullptr) [[unlikely]] { @@ -73,12 +71,12 @@ auto CoroutineAllocator::realloc_memory(void* old_mem, size_t new_size, size_t o return new_mem; } -auto CoroutineAllocator::free_memory(void* mem, size_t size) noexcept -> void { +auto RuntimeCoroutineAllocator::free_memory(void* mem, size_t size) noexcept -> void { kphp::log::assertion(size != 0); memory_resource.deallocate(mem, size); } -auto CoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> void { +auto RuntimeCoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> void { // Extra mem size have to be greater than max chunk block const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; @@ -93,5 +91,3 @@ auto CoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> auto* extra_mem{RuntimeAllocator::get().alloc_global_memory(extra_mem_size)}; memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); } - -} // namespace kphp::coro diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h similarity index 73% rename from runtime-light/allocator/coroutine-allocator.h rename to runtime-light/allocator/runtime-coroutine-allocator.h index 0d6ae39a93..dd0439d724 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -9,13 +9,11 @@ #include "common/mixin/not_copyable.h" #include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" -namespace kphp::coro { +struct RuntimeCoroutineAllocator final : vk::not_copyable { + static auto get() noexcept -> RuntimeCoroutineAllocator&; -struct CoroutineAllocator final : vk::not_copyable { - static auto get() noexcept -> CoroutineAllocator&; - - CoroutineAllocator() noexcept = default; - CoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; + RuntimeCoroutineAllocator() noexcept = default; + RuntimeCoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; void init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept; void free() noexcept; @@ -34,5 +32,3 @@ struct CoroutineAllocator final : vk::not_copyable { private: size_t m_min_extra_mem_size{0}; }; - -} // namespace kphp::coro From 43064baf20487532eddb4084bb9d180b6df31e73 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 10 Aug 2026 19:12:28 +0300 Subject: [PATCH 09/30] add std-like wrapper for coroutine-allocator --- runtime-light/allocator/coroutine-allocator.h | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 runtime-light/allocator/coroutine-allocator.h diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h new file mode 100644 index 0000000000..e508b88e93 --- /dev/null +++ b/runtime-light/allocator/coroutine-allocator.h @@ -0,0 +1,48 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-light/allocator/runtime-coroutine-allocator.h" + +namespace kphp { + +namespace memory { + +template +struct coroutine_allocator { + using value_type = T; + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using is_always_equal = std::true_type; + + coroutine_allocator() noexcept = default; + + template + coroutine_allocator(const coroutine_allocator& /*unused*/) noexcept {} + + constexpr value_type* allocate(size_t n) noexcept { + return static_cast(RuntimeCoroutineAllocator::get().alloc_memory(n * sizeof(T))); + } + + constexpr void deallocate(T* p, size_t n) noexcept { + RuntimeCoroutineAllocator::get().free_memory(p, n * sizeof(T)); + } +}; + +template +constexpr bool operator==(const coroutine_allocator& /*unused*/, const coroutine_allocator& /*unused*/) { + return true; +} + +template +constexpr bool operator!=(const coroutine_allocator& /*unused*/, const coroutine_allocator& /*unused*/) { + return false; +} + +} // namespace memory + +} // namespace kphp From 49c32ca0b76dc2765d2ca28c74fda22bdfa4cd81 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 11 Aug 2026 07:32:24 +0300 Subject: [PATCH 10/30] add runtime-coroutine-allocator to coroutine-instance-state --- runtime-light/allocator/runtime-coroutine-allocator.cpp | 5 +++++ runtime-light/coroutine/coroutine-state.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 5d0eeb36ce..950ea21a1a 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -9,9 +9,14 @@ #include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/allocator/runtime-coroutine-allocator.h" +#include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" +auto RuntimeCoroutineAllocator::get() noexcept -> RuntimeCoroutineAllocator& { + return kphp::coro::instance_state::get().coroutine_allocator; +} + RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index 169ff00993..c043eacf77 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -6,6 +6,7 @@ #include "common/mixin/not_copyable.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/coroutine/async-stack.h" namespace kphp::coro { @@ -17,6 +18,7 @@ struct instance_state final : private vk::not_copyable { static instance_state& get() noexcept; kphp::coro::async_stack_root coroutine_stack_root; + RuntimeCoroutineAllocator coroutine_allocator; }; } // namespace kphp::coro From 59116491a001f47694b4efdad3d5ddb77e7b2670 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 11 Aug 2026 08:29:05 +0300 Subject: [PATCH 11/30] move coroutines related data to coroutine memory --- .../components/kphp/state/instance-state.h | 2 +- runtime-light/coroutine/detail/await-set.h | 14 +++++++------- runtime-light/coroutine/detail/poll-info.h | 6 +++--- .../coroutine/detail/task-self-deleting.h | 8 ++++---- runtime-light/coroutine/detail/when-all.h | 7 ++++--- runtime-light/coroutine/detail/when-any.h | 7 ++++--- runtime-light/coroutine/event.h | 18 ++++++++++++++++-- runtime-light/coroutine/io-scheduler.h | 4 ++-- runtime-light/coroutine/shared-task.h | 8 ++++---- runtime-light/coroutine/task.h | 8 ++++---- .../component/inter-component-session/client.h | 3 ++- runtime-light/stdlib/fork/fork-state.h | 4 ++-- runtime-light/stdlib/fork/wait-queue-state.h | 4 ++-- runtime-light/stdlib/rpc/rpc-client-state.h | 3 ++- runtime-light/stdlib/rpc/rpc-queue-state.h | 4 ++-- 15 files changed, 59 insertions(+), 41 deletions(-) diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index 65ba170383..777b47a800 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -122,7 +122,7 @@ struct InstanceState final : vk::not_copyable { ErrorHandlingState error_handling_instance_state; KmlInstanceState kml_instance_state; - list> shutdown_functions; + kphp::stl::list, kphp::memory::coroutine_allocator> shutdown_functions; private: kphp::coro::task<> init_cli_instance() noexcept; diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 81fb98665e..97a81b27eb 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -11,7 +11,7 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -53,16 +53,16 @@ class await_broker { template void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } void start_task(await_set_task&& task, kphp::coro::async_stack_root& coroutine_stack_root, void* return_address) noexcept { @@ -175,16 +175,16 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { template void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } std::suspend_always initial_suspend() const noexcept { diff --git a/runtime-light/coroutine/detail/poll-info.h b/runtime-light/coroutine/detail/poll-info.h index 30c153a629..852670ae08 100644 --- a/runtime-light/coroutine/detail/poll-info.h +++ b/runtime-light/coroutine/detail/poll-info.h @@ -9,16 +9,16 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/poll.h" #include "runtime-light/k2-platform/k2-api.h" namespace kphp::coro::detail { struct poll_info { - using timed_events = kphp::stl::multimap; - using parked_polls = kphp::stl::multimap; + using timed_events = kphp::stl::multimap; + using parked_polls = kphp::stl::multimap; using scheduled_coroutines = vk::intrusive::list>>; // Each coroutine in the scheduler can be in one of the following states, represented by the `schedule_position` variant: diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index e6908f575b..c5e15fa5ed 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -9,7 +9,7 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -35,16 +35,16 @@ struct promise_self_deleting : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } auto get_return_object() noexcept -> task_self_deleting; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index 2386e4fbb6..da79154105 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -14,6 +14,7 @@ #include #include +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" @@ -152,16 +153,16 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } auto initial_suspend() const noexcept -> std::suspend_always { diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index f5ee01549e..588060f237 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -13,6 +13,7 @@ #include #include +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -162,16 +163,16 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } auto initial_suspend() const noexcept -> std::suspend_always { diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 4be3984ac8..7c2026ec43 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -13,7 +13,7 @@ #include "common/containers/intrusive-list.h" #include "common/mixin/not_copyable.h" #include "common/wrappers/overloaded.h" -#include "runtime-common/core/allocator/script-allocator-managed.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -21,12 +21,26 @@ namespace kphp::coro { class event { - struct event_controller : kphp::memory::script_allocator_managed, vk::not_copyable { + struct event_controller : vk::not_copyable { // 1) std::monostate => not set and no coroutines are waiting // 2) non empty list => linked list of coroutines waiting for the event to trigger // 3) empty list => the event is triggered and all coroutines are resumed std::variant>>> m_state; + template + void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { + return kphp::memory::coro::alloc(n); + } + + template + auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { + return kphp::memory::coro::alloc_aligned(n, al); + } + + void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { + kphp::memory::coro::free(ptr); + } + auto set() noexcept -> void; auto unset() noexcept -> void; auto is_set() const noexcept -> bool; diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 2cc9f01cf8..3579a8ba78 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -22,8 +22,8 @@ #include "common/containers/final_action.h" #include "common/containers/intrusive-list.h" #include "common/wrappers/overloaded.h" -#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -49,7 +49,7 @@ class io_scheduler { kphp::coro::detail::timer_handle m_timer_handle; kphp::coro::detail::poll_info::timed_events m_timed_events; - kphp::stl::vector m_accepted_descriptors; + kphp::stl::vector m_accepted_descriptors; kphp::coro::detail::poll_info::parked_polls m_parked_polls; kphp::coro::detail::poll_info::scheduled_coroutines m_scheduled_coroutines; diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index af6fd6b591..f2958f6b81 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -16,7 +16,7 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -125,16 +125,16 @@ struct promise_base : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } private: diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index d5c064720b..f9a78fd177 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -11,7 +11,7 @@ #include #include "common/containers/final_action.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -66,16 +66,16 @@ struct promise_base : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } void* m_next{}; diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 934b958ed9..fbc9043c11 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -14,6 +14,7 @@ #include #include +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/event.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/component/inter-component-session/details/function-wrapper.h" @@ -27,7 +28,7 @@ namespace kphp::component::inter_component_session { // The client for inter-component communication over a stream in a client-server manner class client final { using query_id_type = uint64_t; - using query2notifier_type = kphp::stl::map; + using query2notifier_type = kphp::stl::map; public: enum class response_readiness : uint8_t { pending, ready }; diff --git a/runtime-light/stdlib/fork/fork-state.h b/runtime-light/stdlib/fork/fork-state.h index d98eddccb5..0bfeff2fb3 100644 --- a/runtime-light/stdlib/fork/fork-state.h +++ b/runtime-light/stdlib/fork/fork-state.h @@ -11,8 +11,8 @@ #include #include "common/mixin/not_copyable.h" -#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/exception-types.h" @@ -40,7 +40,7 @@ struct ForkInstanceState final : private vk::not_copyable { int64_t next_fork_id{FORK_ID_INIT}; // type erased tasks that represent forks - kphp::stl::unordered_map forks; + kphp::stl::unordered_map forks; public: int64_t current_id{FORK_ID_INIT}; diff --git a/runtime-light/stdlib/fork/wait-queue-state.h b/runtime-light/stdlib/fork/wait-queue-state.h index e9eac3e7a9..fd3faacfe5 100644 --- a/runtime-light/stdlib/fork/wait-queue-state.h +++ b/runtime-light/stdlib/fork/wait-queue-state.h @@ -8,14 +8,14 @@ #include #include -#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/await-set.h" class WaitQueueInstanceState { static constexpr int64_t WAIT_QUEUE_ID_INIT = 0; int64_t m_next_wait_queue_id{WAIT_QUEUE_ID_INIT}; - kphp::stl::unordered_map, kphp::memory::script_allocator> m_queues; + kphp::stl::unordered_map, kphp::memory::coroutine_allocator> m_queues; public: WaitQueueInstanceState() noexcept = default; diff --git a/runtime-light/stdlib/rpc/rpc-client-state.h b/runtime-light/stdlib/rpc/rpc-client-state.h index c4d86416b2..22cd9bb97c 100644 --- a/runtime-light/stdlib/rpc/rpc-client-state.h +++ b/runtime-light/stdlib/rpc/rpc-client-state.h @@ -12,6 +12,7 @@ #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/await-set.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/stdlib/rpc/rpc-constants.h" @@ -23,7 +24,7 @@ struct RpcClientInstanceState final : private vk::not_copyable { CurrentTlQuery current_client_query{}; int64_t current_query_id{kphp::rpc::VALID_QUERY_ID_RANGE_START}; - kphp::stl::unordered_map>, kphp::memory::script_allocator> response_awaiter_tasks; + kphp::stl::unordered_map>, kphp::memory::coroutine_allocator> response_awaiter_tasks; kphp::stl::unordered_map, kphp::memory::script_allocator> response_fetcher_instances; kphp::stl::unordered_map, kphp::memory::script_allocator> rpc_responses_extra_info; diff --git a/runtime-light/stdlib/rpc/rpc-queue-state.h b/runtime-light/stdlib/rpc/rpc-queue-state.h index 7c307dd27c..d7598696fa 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-state.h +++ b/runtime-light/stdlib/rpc/rpc-queue-state.h @@ -9,14 +9,14 @@ #include #include "common/mixin/not_copyable.h" -#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/await-set.h" class RpcQueueInstanceState final : private vk::not_copyable { static constexpr int64_t RPC_QUEUE_ID_INIT = 0; int64_t m_rpc_wait_queue_id{RPC_QUEUE_ID_INIT}; - kphp::stl::unordered_map, kphp::memory::script_allocator> m_queues; + kphp::stl::unordered_map, kphp::memory::coroutine_allocator> m_queues; public: RpcQueueInstanceState() noexcept = default; From 9b7ebc2cf5603982320bf2d6cd9c3226c072f1a4 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 11 Aug 2026 09:59:29 +0300 Subject: [PATCH 12/30] add default values and parsing of args for runtime-coroutine-allocator --- runtime-light/allocator/allocator-state.h | 2 -- .../allocator/runtime-coroutine-allocator.cpp | 5 ----- .../components/confdata/bindings/bindings.cpp | 8 +++++++ .../confdata/state/component-state.h | 11 ++++++++++ .../confdata/state/instance-state.h | 7 +++--- .../components/kphp/bindings/bindings.cpp | 8 +++++++ .../components/kphp/state/component-state.cpp | 22 +++++++++++++++++++ .../components/kphp/state/component-state.h | 15 +++++++++++-- .../components/kphp/state/image-state.h | 3 ++- .../components/kphp/state/instance-state.h | 3 +++ runtime-light/coroutine/coroutine-state.h | 2 -- 11 files changed, 71 insertions(+), 15 deletions(-) diff --git a/runtime-light/allocator/allocator-state.h b/runtime-light/allocator/allocator-state.h index a5cc2e79e7..e5dde50264 100644 --- a/runtime-light/allocator/allocator-state.h +++ b/runtime-light/allocator/allocator-state.h @@ -11,8 +11,6 @@ #include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/stdlib/diagnostics/logs.h" -inline constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE{static_cast(1 * 1024U * 1024U)}; // 1Mib - class AllocatorState final : private vk::not_copyable { uint32_t m_libc_alloc_allowed{}; diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 950ea21a1a..5d0eeb36ce 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -9,14 +9,9 @@ #include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/allocator/runtime-coroutine-allocator.h" -#include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" -auto RuntimeCoroutineAllocator::get() noexcept -> RuntimeCoroutineAllocator& { - return kphp::coro::instance_state::get().coroutine_allocator; -} - RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, diff --git a/runtime-light/components/confdata/bindings/bindings.cpp b/runtime-light/components/confdata/bindings/bindings.cpp index 5d3ae4366a..688f40f23c 100644 --- a/runtime-light/components/confdata/bindings/bindings.cpp +++ b/runtime-light/components/confdata/bindings/bindings.cpp @@ -7,6 +7,7 @@ #include "runtime-common/core/runtime-core.h" #include "runtime-light/allocator/allocator-state.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/components/confdata/state/instance-state.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/coroutine/io-scheduler.h" @@ -45,6 +46,13 @@ auto AllocatorState::get() noexcept -> const AllocatorState& { kphp::log::error("can't find allocator state"); } +auto RuntimeCoroutineAllocator::get() noexcept -> RuntimeCoroutineAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->coroutine_allocator; + } + kphp::log::error("can't find runtime coroutine allocator"); +} + auto ErrorHandlingState::try_get() noexcept -> std::optional> { return std::nullopt; // confdata doesn't support PHP error handling } diff --git a/runtime-light/components/confdata/state/component-state.h b/runtime-light/components/confdata/state/component-state.h index c5e3059aa6..2a7e2eddf8 100644 --- a/runtime-light/components/confdata/state/component-state.h +++ b/runtime-light/components/confdata/state/component-state.h @@ -10,9 +10,20 @@ #include "runtime-light/k2-platform/k2-api.h" struct ComponentState final : private vk::not_copyable { + uint64_t initial_instance_memory_size{INIT_INSTANCE_ALLOCATOR_SIZE}; + uint64_t min_instance_extra_memory_size{DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE}; + uint64_t initial_instance_coroutine_memory_size{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; + ComponentState() noexcept = default; static auto get() noexcept -> const ComponentState&; static auto get_mutable() noexcept -> ComponentState&; + +private: + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1U * 1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(2U * 1024U * 1024U); // 2MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(1U * 512U * 1024U); // 0.5MiB }; inline auto ComponentState::get() noexcept -> const ComponentState& { diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 4a7243baea..3530a5c6e4 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -8,6 +8,7 @@ #include "common/mixin/not_copyable.h" #include "runtime-light/allocator/allocator-state.h" +#include "runtime-light/components/confdata/state/component-state.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/task.h" @@ -15,7 +16,9 @@ #include "runtime-light/stdlib/diagnostics/contextual-tags.h" struct InstanceState final : vk::not_copyable { - AllocatorState instance_allocator_state{INIT_INSTANCE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState instance_allocator_state{ComponentState::get().initial_instance_memory_size, ComponentState::get().min_instance_extra_memory_size, 0}; + RuntimeCoroutineAllocator coroutine_allocator{ComponentState::get().initial_instance_coroutine_memory_size, + ComponentState::get().min_instance_extra_coroutine_memory_size, 0}; kphp::log::contextual_tags instance_tags; @@ -28,8 +31,6 @@ struct InstanceState final : vk::not_copyable { auto init() noexcept -> void; private: - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB - auto run() noexcept -> kphp::coro::task<>; }; diff --git a/runtime-light/components/kphp/bindings/bindings.cpp b/runtime-light/components/kphp/bindings/bindings.cpp index 5f1b40b704..cafebdae58 100644 --- a/runtime-light/components/kphp/bindings/bindings.cpp +++ b/runtime-light/components/kphp/bindings/bindings.cpp @@ -8,6 +8,7 @@ #include "runtime-common/core/runtime-core.h" #include "runtime-light/allocator/allocator-state.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/components/kphp/state/component-state.h" #include "runtime-light/components/kphp/state/image-state.h" #include "runtime-light/components/kphp/state/instance-state.h" @@ -76,6 +77,13 @@ auto AllocatorState::get() noexcept -> const AllocatorState& { kphp::log::error("can't find allocator state"); } +auto RuntimeCoroutineAllocator::get() noexcept -> RuntimeCoroutineAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->coroutine_allocator; + } + kphp::log::error("can't find runtime coroutine allocator"); +} + auto RuntimeContext::get() noexcept -> RuntimeContext& { if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { return instance_state_ptr->runtime_context; diff --git a/runtime-light/components/kphp/state/component-state.cpp b/runtime-light/components/kphp/state/component-state.cpp index 9cd5b6e9da..251f73d35d 100644 --- a/runtime-light/components/kphp/state/component-state.cpp +++ b/runtime-light/components/kphp/state/component-state.cpp @@ -132,6 +132,24 @@ void ComponentState::parse_min_instance_extra_memory_size_arg(std::string_view v kphp::log::info("set min instance extra memory size to {} bytes", min_instance_extra_memory_size); } +void ComponentState::parse_initial_instance_coroutine_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse initial instance coroutine memory size, got {}", value_view); + } + initial_instance_coroutine_memory_size = *parsed; + kphp::log::info("set initial instance coroutine memory size to {} bytes", initial_instance_coroutine_memory_size); +} + +void ComponentState::parse_min_instance_extra_coroutine_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse min instance extra coroutine memory size, got {}", value_view); + } + min_instance_extra_coroutine_memory_size = *parsed; + kphp::log::info("set min instance extra coroutine memory size to {} bytes", min_instance_extra_coroutine_memory_size); +} + void ComponentState::parse_args() noexcept { for (auto i = 0; i < argc; ++i) { const auto [arg_key, arg_value]{k2::arg_fetch(i)}; @@ -152,6 +170,10 @@ void ComponentState::parse_args() noexcept { parse_initial_instance_memory_size_arg(value_view); } else if (key_view == MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG) { parse_min_instance_extra_memory_size_arg(value_view); + } else if (key_view == INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG) { + parse_initial_instance_coroutine_memory_size_arg(value_view); + } else if (key_view == MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG) { + parse_min_instance_extra_coroutine_memory_size_arg(value_view); } else { kphp::log::warning("unexpected argument format: {}", key_view); } diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index 9182b62107..987ca7129b 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -31,6 +31,8 @@ struct ComponentState final : private vk::not_copyable { bool exit_after_response{}; uint64_t initial_instance_memory_size{INIT_INSTANCE_ALLOCATOR_SIZE}; uint64_t min_instance_extra_memory_size{DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE}; + uint64_t initial_instance_coroutine_memory_size{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; ComponentState() noexcept { parse_env(); @@ -63,8 +65,13 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view EXIT_AFTER_RESPONSE_ARG = "exit-after-response"; static constexpr std::string_view INITIAL_INSTANCE_MEMORY_SIZE_ARG = "initial-instance-memory-size"; static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG = "min-instance-extra-memory-size"; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr std::string_view INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG = "initial-instance-coroutine-memory-size"; + static constexpr std::string_view MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG = "min-instance-extra-coroutine-memory-size"; + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1U * 1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(1U * 1024U * 1024U); // 1MiB void parse_env() noexcept; @@ -83,4 +90,8 @@ struct ComponentState final : private vk::not_copyable { void parse_initial_instance_memory_size_arg(std::string_view) noexcept; void parse_min_instance_extra_memory_size_arg(std::string_view) noexcept; + + void parse_initial_instance_coroutine_memory_size_arg(std::string_view) noexcept; + + void parse_min_instance_extra_coroutine_memory_size_arg(std::string_view) noexcept; }; diff --git a/runtime-light/components/kphp/state/image-state.h b/runtime-light/components/kphp/state/image-state.h index 7d3fabae02..6245df78dd 100644 --- a/runtime-light/components/kphp/state/image-state.h +++ b/runtime-light/components/kphp/state/image-state.h @@ -102,5 +102,6 @@ struct ImageState final : private vk::not_copyable { } private: - static constexpr auto INIT_IMAGE_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_IMAGE_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB }; diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index 777b47a800..c8856d2d0d 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -11,6 +11,7 @@ #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" #include "runtime-light/allocator/allocator-state.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/components/kphp/state/component-state.h" #include "runtime-light/core/globals/php-script-globals.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -89,6 +90,8 @@ struct InstanceState final : vk::not_copyable { } AllocatorState instance_allocator_state{ComponentState::get().initial_instance_memory_size, ComponentState::get().min_instance_extra_memory_size, 0}; + RuntimeCoroutineAllocator coroutine_allocator{ComponentState::get().initial_instance_coroutine_memory_size, + ComponentState::get().min_instance_extra_coroutine_memory_size, 0}; kphp::log::contextual_tags instance_tags; diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index c043eacf77..169ff00993 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -6,7 +6,6 @@ #include "common/mixin/not_copyable.h" -#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/coroutine/async-stack.h" namespace kphp::coro { @@ -18,7 +17,6 @@ struct instance_state final : private vk::not_copyable { static instance_state& get() noexcept; kphp::coro::async_stack_root coroutine_stack_root; - RuntimeCoroutineAllocator coroutine_allocator; }; } // namespace kphp::coro From 3b1ed037d3b2ccecbebf9f8c4e1075fc08513323 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 11 Aug 2026 10:12:34 +0300 Subject: [PATCH 13/30] add includes --- runtime-light/allocator/coroutine-allocator.h | 1 + runtime-light/allocator/coroutine-malloc-interface.h | 1 + 2 files changed, 2 insertions(+) diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h index e508b88e93..ca299166a7 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/coroutine-allocator.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include "runtime-light/allocator/runtime-coroutine-allocator.h" diff --git a/runtime-light/allocator/coroutine-malloc-interface.h b/runtime-light/allocator/coroutine-malloc-interface.h index e05faaa5b4..8c48b1bd77 100644 --- a/runtime-light/allocator/coroutine-malloc-interface.h +++ b/runtime-light/allocator/coroutine-malloc-interface.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "common/wrappers/likely.h" #include "runtime-common/core/allocator/detail/control-block.h" From 81bde557df669fa0838e2c19991d3d05ffe59584 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 12:48:50 +0300 Subject: [PATCH 14/30] change default values for memory size --- .../components/confdata/state/component-state.h | 8 ++++---- runtime-light/components/kphp/state/component-state.h | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/runtime-light/components/confdata/state/component-state.h b/runtime-light/components/confdata/state/component-state.h index 2a7e2eddf8..a0612da5cd 100644 --- a/runtime-light/components/confdata/state/component-state.h +++ b/runtime-light/components/confdata/state/component-state.h @@ -20,10 +20,10 @@ struct ComponentState final : private vk::not_copyable { static auto get_mutable() noexcept -> ComponentState&; private: - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB - static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1U * 1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(2U * 1024U * 1024U); // 2MiB - static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(1U * 512U * 1024U); // 0.5MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(2U * 1024U * 1024U); // 2MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(512U * 1024U); // 0.5MiB }; inline auto ComponentState::get() noexcept -> const ComponentState& { diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index 987ca7129b..e66760ca66 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -67,11 +67,11 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG = "min-instance-extra-memory-size"; static constexpr std::string_view INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG = "initial-instance-coroutine-memory-size"; static constexpr std::string_view MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG = "min-instance-extra-coroutine-memory-size"; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB - static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1U * 1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB - static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(1U * 1024U * 1024U); // 1MiB + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB void parse_env() noexcept; From 4ce05f3e8f9908e780e3fcabebe4f3efe10dce2f Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 14:33:34 +0300 Subject: [PATCH 15/30] enlarge extra-coroutine-memory-pool-size value to 4MB --- runtime-light/components/kphp/state/component-state.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index e66760ca66..0f72f4b418 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -67,11 +67,11 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG = "min-instance-extra-memory-size"; static constexpr std::string_view INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG = "initial-instance-coroutine-memory-size"; static constexpr std::string_view MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG = "min-instance-extra-coroutine-memory-size"; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB - static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB - static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB void parse_env() noexcept; From d4286f5fdc4394ec1359d812a0e7128d9ed26dfc Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 16:00:17 +0300 Subject: [PATCH 16/30] small fixes --- runtime-light/allocator/coroutine-allocator.h | 3 --- .../allocator/runtime-coroutine-allocator.cpp | 11 ++++++++--- runtime-light/allocator/runtime-coroutine-allocator.h | 6 ++++-- runtime-light/components/kphp/state/instance-state.h | 3 --- runtime-light/coroutine/event.h | 6 ++---- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h index ca299166a7..ae9822fa2f 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/coroutine-allocator.h @@ -16,9 +16,6 @@ namespace memory { template struct coroutine_allocator { using value_type = T; - using propagate_on_container_copy_assignment = std::true_type; - using propagate_on_container_move_assignment = std::true_type; - using is_always_equal = std::true_type; coroutine_allocator() noexcept = default; diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 5d0eeb36ce..547f4ea688 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -7,7 +7,6 @@ #include #include -#include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -16,7 +15,7 @@ RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t mem_size, size_t min : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, // oom_handling_mem_size); - void* buffer{RuntimeAllocator::get().alloc_global_memory(mem_size)}; + void* buffer{alloc_global_memory(mem_size)}; memory_resource.init(buffer, mem_size, oom_handling_mem_size); } @@ -76,6 +75,12 @@ auto RuntimeCoroutineAllocator::free_memory(void* mem, size_t size) noexcept -> memory_resource.deallocate(mem, size); } +auto RuntimeCoroutineAllocator::alloc_global_memory(size_t size) noexcept -> void* { + void* mem{k2::alloc(size)}; + kphp::log::assertion(mem != nullptr); + return mem; +} + auto RuntimeCoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> void { // Extra mem size have to be greater than max chunk block const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; @@ -88,6 +93,6 @@ auto RuntimeCoroutineAllocator::request_extra_memory(size_t requested_size) noex // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); - auto* extra_mem{RuntimeAllocator::get().alloc_global_memory(extra_mem_size)}; + auto* extra_mem{alloc_global_memory(extra_mem_size)}; memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); } diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index dd0439d724..4fca1127b6 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -15,8 +15,8 @@ struct RuntimeCoroutineAllocator final : vk::not_copyable { RuntimeCoroutineAllocator() noexcept = default; RuntimeCoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; - void init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept; - void free() noexcept; + auto init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; auto alloc_memory(size_t size) noexcept -> void*; auto alloc0_memory(size_t size) noexcept -> void*; @@ -24,6 +24,8 @@ struct RuntimeCoroutineAllocator final : vk::not_copyable { auto free_memory(void* mem, size_t size) noexcept -> void; private: + auto alloc_global_memory(size_t size) noexcept -> void*; + auto request_extra_memory(size_t requested_size) noexcept -> void; public: diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index c8856d2d0d..c2979fa4d5 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -61,9 +61,6 @@ struct InstanceState final : vk::not_copyable { template using deque = kphp::stl::deque; - template - using list = kphp::stl::list; - // It's important to use `{}` instead of `= default` here. // In the second case clang++ zeroes the whole structure. // It drastically ruins performance. Be careful! diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 7c2026ec43..5eec557c4d 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -27,13 +27,11 @@ class event { // 3) empty list => the event is triggered and all coroutines are resumed std::variant>>> m_state; - template - void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { + void* operator new(size_t n) noexcept { return kphp::memory::coro::alloc(n); } - template - auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { + auto operator new(size_t n, std::align_val_t al) noexcept -> void* { return kphp::memory::coro::alloc_aligned(n, al); } From 10d174f328c71e509187d0ab83d38e4edb3d9213 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 16:10:50 +0300 Subject: [PATCH 17/30] add methods to allocate global memory for coroutines --- .../allocator/runtime-coroutine-allocator.cpp | 17 +++++++++++++++++ .../allocator/runtime-coroutine-allocator.h | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 547f4ea688..a1302941c5 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -81,6 +81,23 @@ auto RuntimeCoroutineAllocator::alloc_global_memory(size_t size) noexcept -> voi return mem; } +void* RuntimeCoroutineAllocator::alloc0_global_memory(size_t size) noexcept { + void* mem{k2::alloc(size)}; + kphp::log::assertion(mem != nullptr); + std::memset(mem, 0, size); + return mem; +} + +void* RuntimeCoroutineAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept { + void* mem{k2::realloc(old_mem, new_size)}; + kphp::log::assertion(mem != nullptr); + return mem; +} + +void RuntimeCoroutineAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept { + k2::free(mem); +} + auto RuntimeCoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> void { // Extra mem size have to be greater than max chunk block const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index 4fca1127b6..63842f41e5 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -23,9 +23,12 @@ struct RuntimeCoroutineAllocator final : vk::not_copyable { auto realloc_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_memory(void* mem, size_t size) noexcept -> void; -private: auto alloc_global_memory(size_t size) noexcept -> void*; + auto alloc0_global_memory(size_t size) noexcept -> void*; + auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_global_memory(void* mem, size_t size) noexcept -> void; +private: auto request_extra_memory(size_t requested_size) noexcept -> void; public: From 7b5b08deee87f562ea0984e8c1f260caab149a66 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 17:16:22 +0300 Subject: [PATCH 18/30] reduce code duplication --- .../core/allocator/detail/control-block.h | 48 ----- .../core/allocator/details/malloc-interface.h | 173 ++++++++++++++++++ .../core/allocator/details/pool-allocator.h | 41 +++++ .../core/allocator/runtime-allocator.h | 32 +--- .../core/allocator/script-malloc-interface.h | 113 ++---------- runtime-light/allocator/allocator.cmake | 2 +- runtime-light/allocator/coroutine-allocator.h | 4 +- .../allocator/coroutine-malloc-interface.h | 118 +++--------- .../pool-allocator.cpp} | 44 +++-- .../allocator/runtime-coroutine-allocator.h | 32 +--- .../allocator/runtime-light-allocator.cpp | 110 ----------- 11 files changed, 284 insertions(+), 433 deletions(-) delete mode 100644 runtime-common/core/allocator/detail/control-block.h create mode 100644 runtime-common/core/allocator/details/malloc-interface.h create mode 100644 runtime-common/core/allocator/details/pool-allocator.h rename runtime-light/allocator/{runtime-coroutine-allocator.cpp => details/pool-allocator.cpp} (60%) diff --git a/runtime-common/core/allocator/detail/control-block.h b/runtime-common/core/allocator/detail/control-block.h deleted file mode 100644 index d41bb6040c..0000000000 --- a/runtime-common/core/allocator/detail/control-block.h +++ /dev/null @@ -1,48 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2026 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include -#include - -namespace kphp::memory::detail { - -struct control_block { -private: - static constexpr auto SIZE_FIELD_BITSIZE{48}; - static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; - static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; - static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; - - static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); - -public: - static constexpr uint64_t max_size() noexcept { - return 1UL << SIZE_FIELD_BITSIZE; - } - - static constexpr uint64_t max_alignment() noexcept { - return 1UL << BASE_OFFSET_FIELD_BITSIZE; - } - - uint64_t raw() const noexcept { - return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); - } - - static control_block from_raw(uint64_t raw) noexcept { - return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; - } - - uint64_t size : SIZE_FIELD_BITSIZE; - uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; -}; - -inline bool is_power_of_2(uint64_t v) noexcept { - return v && !(v & (v - 1)); -} - -static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); - -} // namespace kphp::memory::detail diff --git a/runtime-common/core/allocator/details/malloc-interface.h b/runtime-common/core/allocator/details/malloc-interface.h new file mode 100644 index 0000000000..ab491507ac --- /dev/null +++ b/runtime-common/core/allocator/details/malloc-interface.h @@ -0,0 +1,173 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/wrappers/likely.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace kphp { + +namespace memory { + +namespace details { + +struct control_block { +private: + static constexpr auto SIZE_FIELD_BITSIZE{48}; + static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; + static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; + static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; + + static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); + +public: + static constexpr uint64_t max_size() noexcept { + return 1UL << SIZE_FIELD_BITSIZE; + } + + static constexpr uint64_t max_alignment() noexcept { + return 1UL << BASE_OFFSET_FIELD_BITSIZE; + } + + uint64_t raw() const noexcept { + return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); + } + + static control_block from_raw(uint64_t raw) noexcept { + return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; + } + + uint64_t size : SIZE_FIELD_BITSIZE; + uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; +}; + +inline bool is_power_of_2(uint64_t v) noexcept { + return v && !(v & (v - 1)); +} + +static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); + +constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB + +template +struct malloc_interface { + static auto alloc(size_t size) noexcept -> void* { + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + if (unlikely(size > std::min(kphp::memory::details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + const size_t total_size{size + cb_size}; + void* base{AllocatorGetter().alloc_script_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + *(static_cast(base)) = kphp::memory::details::control_block{.size = total_size, .base_offset = cb_size}.raw(); + return static_cast(static_cast(base) + cb_size); + } + + static auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + // Check that provided alignment is power of two + const size_t align{static_cast(alignment)}; + if (unlikely(align == 0 || !kphp::memory::details::is_power_of_2(align) || align >= kphp::memory::details::control_block::max_alignment())) { + php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", + kphp::memory::details::control_block::max_alignment(), align); + return nullptr; + } + + // Check that memory is enough + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + if (unlikely(size > std::min(kphp::memory::details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + + // Request mem from underlying memory manager + const size_t total_size{size + (align - 1) + cb_size}; + void* base{AllocatorGetter().alloc_script_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + + const uint64_t base_u{reinterpret_cast(base)}; + // The smallest multiple of `align` greater than or equal to requested memory + const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; + const uint64_t base_offset_u{aligned_u - base_u}; + + // Save control block + *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT + kphp::memory::details::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); + + return reinterpret_cast(aligned_u); // NOLINT + } + + static auto calloc(size_t num, size_t size) noexcept -> void* { + void* ptr{alloc(num * size)}; + if (unlikely(ptr == nullptr)) { + return nullptr; + } + return std::memset(ptr, 0, num * size); + } + + static auto free(void* ptr) noexcept -> void { + if (unlikely(ptr == nullptr)) { + return; + } + + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + + AllocatorGetter().free_script_memory(base, cb.size); + } + + static auto realloc(void* ptr, size_t new_size) noexcept -> void* { + if (unlikely(ptr == nullptr)) { + return alloc(new_size); + } + + if (unlikely(new_size == 0)) { + free(ptr); + return nullptr; + } + + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + + void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + const size_t old_size{cb.size}; + + void* new_ptr{alloc(new_size)}; + if (likely(new_ptr != nullptr)) { + std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); + AllocatorGetter().free_script_memory(old_base, old_size); + } + return new_ptr; + } + + static auto strdup(const char* str1) noexcept -> char* { + auto* str2{static_cast(alloc(std::strlen(str1) + 1))}; + return std::strcpy(str2, str1); + } +}; + +} // namespace details + +} // namespace memory + +} // namespace kphp diff --git a/runtime-common/core/allocator/details/pool-allocator.h b/runtime-common/core/allocator/details/pool-allocator.h new file mode 100644 index 0000000000..a11296e5ca --- /dev/null +++ b/runtime-common/core/allocator/details/pool-allocator.h @@ -0,0 +1,41 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" + +namespace kphp::memory::details { + +struct PoolAllocator : vk::not_copyable { + PoolAllocator() = default; + PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size); + + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) -> void; + auto free() -> void; + + auto alloc_script_memory(size_t size) noexcept -> void*; + auto alloc0_script_memory(size_t size) noexcept -> void*; + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_script_memory(void* mem, size_t size) noexcept -> void; + + auto alloc_global_memory(size_t size) noexcept -> void*; + auto alloc0_global_memory(size_t size) noexcept -> void*; + auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_global_memory(void* mem, size_t size) noexcept -> void; + +private: + auto request_extra_memory(size_t requested_size) noexcept -> void; + +public: + memory_resource::unsynchronized_pool_resource memory_resource; + +private: + size_t m_min_extra_mem_size{0}; +}; + +} // namespace kphp::memory::details diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index aac9d859d8..a121cdfae4 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -6,34 +6,10 @@ #include -#include "common/mixin/not_copyable.h" -#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" +#include "runtime-common/core/allocator/details/pool-allocator.h" -struct RuntimeAllocator final : vk::not_copyable { - static RuntimeAllocator& get() noexcept; - - RuntimeAllocator() = default; - RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size); - - void init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size); - void free(); - - void* alloc_script_memory(size_t size) noexcept; - void* alloc0_script_memory(size_t size) noexcept; - void* realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept; - void free_script_memory(void* mem, size_t size) noexcept; +struct RuntimeAllocator final : public kphp::memory::details::PoolAllocator { + using kphp::memory::details::PoolAllocator::PoolAllocator; - void* alloc_global_memory(size_t size) noexcept; - void* alloc0_global_memory(size_t size) noexcept; - void* realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept; - void free_global_memory(void* mem, size_t size) noexcept; - -private: - void request_extra_memory(size_t requested_size) noexcept; - -public: - memory_resource::unsynchronized_pool_resource memory_resource; - -private: - size_t m_min_extra_mem_size{0}; + static RuntimeAllocator& get() noexcept; }; diff --git a/runtime-common/core/allocator/script-malloc-interface.h b/runtime-common/core/allocator/script-malloc-interface.h index 638c01be8a..944c88b03f 100644 --- a/runtime-common/core/allocator/script-malloc-interface.h +++ b/runtime-common/core/allocator/script-malloc-interface.h @@ -4,15 +4,11 @@ #pragma once -#include #include -#include #include -#include "common/wrappers/likely.h" -#include "runtime-common/core/allocator/detail/control-block.h" +#include "runtime-common/core/allocator/details/malloc-interface.h" #include "runtime-common/core/allocator/runtime-allocator.h" -#include "runtime-common/core/utils/kphp-assert-core.h" namespace kphp { @@ -20,111 +16,28 @@ namespace memory { namespace script { -constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB - -inline void* alloc(size_t size) noexcept { - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - const size_t total_size{size + cb_size}; - void* base{RuntimeAllocator::get().alloc_script_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - *(static_cast(base)) = kphp::memory::detail::control_block{.size = total_size, .base_offset = cb_size}.raw(); - return static_cast(static_cast(base) + cb_size); +inline auto alloc(size_t size) noexcept -> void* { + return details::malloc_interface::alloc(size); } -inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { - // Check that provided alignment is power of two - const size_t align{static_cast(alignment)}; - if (unlikely(align == 0 || !kphp::memory::detail::is_power_of_2(align) || align >= kphp::memory::detail::control_block::max_alignment())) { - php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", - kphp::memory::detail::control_block::max_alignment(), align); - return nullptr; - } - - // Check that memory is enough - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - - // Request mem from underlying memory manager - const size_t total_size{size + (align - 1) + cb_size}; - void* base{RuntimeAllocator::get().alloc_script_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - - const uint64_t base_u{reinterpret_cast(base)}; - // The smallest multiple of `align` greater than or equal to requested memory - const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; - const uint64_t base_offset_u{aligned_u - base_u}; - - // Save control block - *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT - kphp::memory::detail::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); - - return reinterpret_cast(aligned_u); // NOLINT +inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + return details::malloc_interface::alloc_aligned(size, alignment); } -inline void* calloc(size_t num, size_t size) noexcept { - void* ptr{kphp::memory::script::alloc(num * size)}; - if (unlikely(ptr == nullptr)) { - return nullptr; - } - return std::memset(ptr, 0, num * size); +inline auto calloc(size_t num, size_t size) noexcept -> void* { + return details::malloc_interface::calloc(num, size); } -inline void free(void* ptr) noexcept { - if (unlikely(ptr == nullptr)) { - return; - } - - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - const auto mem{reinterpret_cast(ptr)}; - - const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - - RuntimeAllocator::get().free_script_memory(base, cb.size); +inline auto free(void* ptr) noexcept -> void { + details::malloc_interface::free(ptr); } -inline void* realloc(void* ptr, size_t new_size) noexcept { - if (unlikely(ptr == nullptr)) { - return kphp::memory::script::alloc(new_size); - } - - if (unlikely(new_size == 0)) { - kphp::memory::script::free(ptr); - return nullptr; - } - - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - const auto mem{reinterpret_cast(ptr)}; - - const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - - void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - const size_t old_size{cb.size}; - - void* new_ptr{kphp::memory::script::alloc(new_size)}; - if (likely(new_ptr != nullptr)) { - std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - RuntimeAllocator::get().free_script_memory(old_base, old_size); - } - return new_ptr; +inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { + return details::malloc_interface::realloc(ptr, new_size); } -inline char* strdup(const char* str1) noexcept { - auto* str2{static_cast(kphp::memory::script::alloc(std::strlen(str1) + 1))}; - return std::strcpy(str2, str1); +inline auto strdup(const char* str1) noexcept -> char* { + return details::malloc_interface::strdup(str1); } } // namespace script diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake index 777c6117b6..85880d002e 100644 --- a/runtime-light/allocator/allocator.cmake +++ b/runtime-light/allocator/allocator.cmake @@ -1 +1 @@ -set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp allocator/runtime-coroutine-allocator.cpp) +set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp) diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h index ae9822fa2f..29e576653a 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/coroutine-allocator.h @@ -23,11 +23,11 @@ struct coroutine_allocator { coroutine_allocator(const coroutine_allocator& /*unused*/) noexcept {} constexpr value_type* allocate(size_t n) noexcept { - return static_cast(RuntimeCoroutineAllocator::get().alloc_memory(n * sizeof(T))); + return static_cast(RuntimeCoroutineAllocator::get().alloc_script_memory(n * sizeof(T))); } constexpr void deallocate(T* p, size_t n) noexcept { - RuntimeCoroutineAllocator::get().free_memory(p, n * sizeof(T)); + RuntimeCoroutineAllocator::get().free_script_memory(p, n * sizeof(T)); } }; diff --git a/runtime-light/allocator/coroutine-malloc-interface.h b/runtime-light/allocator/coroutine-malloc-interface.h index 8c48b1bd77..47b7561a9b 100644 --- a/runtime-light/allocator/coroutine-malloc-interface.h +++ b/runtime-light/allocator/coroutine-malloc-interface.h @@ -4,118 +4,44 @@ #pragma once -#include #include -#include #include -#include "common/wrappers/likely.h" -#include "runtime-common/core/allocator/detail/control-block.h" -#include "runtime-common/core/utils/kphp-assert-core.h" +#include "runtime-common/core/allocator/details/malloc-interface.h" #include "runtime-light/allocator/runtime-coroutine-allocator.h" -namespace kphp::memory::coro { +namespace kphp { -constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB +namespace memory { -inline void* alloc(size_t size) noexcept { - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - const size_t total_size{size + cb_size}; - void* base{RuntimeCoroutineAllocator::get().alloc_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough coroutine memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - *(static_cast(base)) = kphp::memory::detail::control_block{.size = total_size, .base_offset = cb_size}.raw(); - return static_cast(static_cast(base) + cb_size); -} - -inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { - // Check that provided alignment is power of two - const size_t align{static_cast(alignment)}; - if (unlikely(align == 0 || !kphp::memory::detail::is_power_of_2(align) || align >= kphp::memory::detail::control_block::max_alignment())) { - php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", - kphp::memory::detail::control_block::max_alignment(), align); - return nullptr; - } - - // Check that memory is enough - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - if (unlikely(size > std::min(kphp::memory::detail::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - - // Request mem from underlying memory manager - const size_t total_size{size + (align - 1) + cb_size}; - void* base{RuntimeCoroutineAllocator::get().alloc_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough coroutine memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } +namespace coro { - const uint64_t base_u{reinterpret_cast(base)}; - // The smallest multiple of `align` greater than or equal to requested memory - const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; - const uint64_t base_offset_u{aligned_u - base_u}; - - // Save control block - *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT - kphp::memory::detail::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); - - return reinterpret_cast(aligned_u); // NOLINT +inline auto alloc(size_t size) noexcept -> void* { + return details::malloc_interface::alloc(size); } -inline void* calloc(size_t num, size_t size) noexcept { - void* ptr{kphp::memory::coro::alloc(num * size)}; - if (unlikely(ptr == nullptr)) { - return nullptr; - } - return std::memset(ptr, 0, num * size); +inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + return details::malloc_interface::alloc_aligned(size, alignment); } -inline void free(void* ptr) noexcept { - if (unlikely(ptr == nullptr)) { - return; - } - - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - const auto mem{reinterpret_cast(ptr)}; - - const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - - RuntimeCoroutineAllocator::get().free_memory(base, cb.size); +inline auto calloc(size_t num, size_t size) noexcept -> void* { + return details::malloc_interface::calloc(num, size); } -inline void* realloc(void* ptr, size_t new_size) noexcept { - if (unlikely(ptr == nullptr)) { - return kphp::memory::coro::alloc(new_size); - } - - if (unlikely(new_size == 0)) { - kphp::memory::coro::free(ptr); - return nullptr; - } +inline auto free(void* ptr) noexcept -> void { + details::malloc_interface::free(ptr); +} - constexpr size_t cb_size{sizeof(kphp::memory::detail::control_block)}; - const auto mem{reinterpret_cast(ptr)}; +inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { + return details::malloc_interface::realloc(ptr, new_size); +} - const auto cb{kphp::memory::detail::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT +inline auto strdup(const char* str1) noexcept -> char* { + return details::malloc_interface::strdup(str1); +} - void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - const size_t old_size{cb.size}; +} // namespace coro - void* new_ptr{kphp::memory::coro::alloc(new_size)}; - if (likely(new_ptr != nullptr)) { - std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - RuntimeCoroutineAllocator::get().free_memory(old_base, old_size); - } - return new_ptr; -} +} // namespace memory -} // namespace kphp::memory::coro +} // namespace kphp diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/details/pool-allocator.cpp similarity index 60% rename from runtime-light/allocator/runtime-coroutine-allocator.cpp rename to runtime-light/allocator/details/pool-allocator.cpp index a1302941c5..cece64716c 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/details/pool-allocator.cpp @@ -7,27 +7,29 @@ #include #include -#include "runtime-light/allocator/runtime-coroutine-allocator.h" +#include "runtime-common/core/allocator/details/pool-allocator.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" -RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept +namespace kphp::memory::details { + +PoolAllocator::PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) : m_min_extra_mem_size(min_extra_mem_size) { - // kphp::log::debug("create coroutine allocator -> {:p}: memory -> {}, oom handling size -> {}", reinterpret_cast(this), mem_size, + // kphp::log::debug("create pool allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, // oom_handling_mem_size); - void* buffer{alloc_global_memory(mem_size)}; - memory_resource.init(buffer, mem_size, oom_handling_mem_size); + void* buffer{alloc_global_memory(script_mem_size)}; + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); } -auto RuntimeCoroutineAllocator::init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept -> void { +auto PoolAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) -> void { kphp::log::assertion(buffer != nullptr); - // kphp::log::debug("init coroutine allocator -> {:p}: buffer -> {:p}, memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, - // mem_size, oom_handling_mem_size); - memory_resource.init(buffer, mem_size, oom_handling_mem_size); + // kphp::log::debug("init pool allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, + // script_mem_size, oom_handling_mem_size); + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); } -auto RuntimeCoroutineAllocator::free() noexcept -> void { - // kphp::log::debug("free coroutine allocator -> {:p}", reinterpret_cast(this)); +auto PoolAllocator::free() -> void { + // kphp::log::debug("free pool allocator -> {:p}", reinterpret_cast(this)); auto* extra_memory{memory_resource.get_extra_memory_head()}; while (extra_memory->get_pool_payload_size() != 0) { auto* extra_memory_to_release{extra_memory}; @@ -37,7 +39,7 @@ auto RuntimeCoroutineAllocator::free() noexcept -> void { k2::free(memory_resource.memory_begin()); } -auto RuntimeCoroutineAllocator::alloc_memory(size_t size) noexcept -> void* { +auto PoolAllocator::alloc_script_memory(size_t size) noexcept -> void* { kphp::log::assertion(size != 0); void* mem{memory_resource.allocate(size)}; if (mem == nullptr) [[unlikely]] { @@ -48,7 +50,7 @@ auto RuntimeCoroutineAllocator::alloc_memory(size_t size) noexcept -> void* { return mem; } -auto RuntimeCoroutineAllocator::alloc0_memory(size_t size) noexcept -> void* { +auto PoolAllocator::alloc0_script_memory(size_t size) noexcept -> void* { kphp::log::assertion(size != 0); void* mem{memory_resource.allocate0(size)}; if (mem == nullptr) [[unlikely]] { @@ -59,7 +61,7 @@ auto RuntimeCoroutineAllocator::alloc0_memory(size_t size) noexcept -> void* { return mem; } -auto RuntimeCoroutineAllocator::realloc_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { +auto PoolAllocator::realloc_script_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { kphp::log::assertion(new_size > old_size); void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; if (new_mem == nullptr) [[unlikely]] { @@ -70,35 +72,35 @@ auto RuntimeCoroutineAllocator::realloc_memory(void* old_mem, size_t new_size, s return new_mem; } -auto RuntimeCoroutineAllocator::free_memory(void* mem, size_t size) noexcept -> void { +auto PoolAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { kphp::log::assertion(size != 0); memory_resource.deallocate(mem, size); } -auto RuntimeCoroutineAllocator::alloc_global_memory(size_t size) noexcept -> void* { +auto PoolAllocator::alloc_global_memory(size_t size) noexcept -> void* { void* mem{k2::alloc(size)}; kphp::log::assertion(mem != nullptr); return mem; } -void* RuntimeCoroutineAllocator::alloc0_global_memory(size_t size) noexcept { +auto PoolAllocator::alloc0_global_memory(size_t size) noexcept -> void* { void* mem{k2::alloc(size)}; kphp::log::assertion(mem != nullptr); std::memset(mem, 0, size); return mem; } -void* RuntimeCoroutineAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept { +auto PoolAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept -> void* { void* mem{k2::realloc(old_mem, new_size)}; kphp::log::assertion(mem != nullptr); return mem; } -void RuntimeCoroutineAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept { +auto PoolAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept -> void { k2::free(mem); } -auto RuntimeCoroutineAllocator::request_extra_memory(size_t requested_size) noexcept -> void { +auto PoolAllocator::request_extra_memory(size_t requested_size) noexcept -> void { // Extra mem size have to be greater than max chunk block const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; @@ -113,3 +115,5 @@ auto RuntimeCoroutineAllocator::request_extra_memory(size_t requested_size) noex auto* extra_mem{alloc_global_memory(extra_mem_size)}; memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); } + +} // namespace kphp::memory::details diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index 63842f41e5..f45a59e704 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -6,34 +6,10 @@ #include -#include "common/mixin/not_copyable.h" -#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" +#include "runtime-common/core/allocator/details/pool-allocator.h" -struct RuntimeCoroutineAllocator final : vk::not_copyable { - static auto get() noexcept -> RuntimeCoroutineAllocator&; - - RuntimeCoroutineAllocator() noexcept = default; - RuntimeCoroutineAllocator(size_t mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; - - auto init(void* buffer, size_t mem_size, size_t oom_handling_mem_size) noexcept -> void; - auto free() noexcept -> void; - - auto alloc_memory(size_t size) noexcept -> void*; - auto alloc0_memory(size_t size) noexcept -> void*; - auto realloc_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; - auto free_memory(void* mem, size_t size) noexcept -> void; +struct RuntimeCoroutineAllocator final : public kphp::memory::details::PoolAllocator { + using kphp::memory::details::PoolAllocator::PoolAllocator; - auto alloc_global_memory(size_t size) noexcept -> void*; - auto alloc0_global_memory(size_t size) noexcept -> void*; - auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; - auto free_global_memory(void* mem, size_t size) noexcept -> void; - -private: - auto request_extra_memory(size_t requested_size) noexcept -> void; - -public: - memory_resource::unsynchronized_pool_resource memory_resource; - -private: - size_t m_min_extra_mem_size{0}; + static auto get() noexcept -> RuntimeCoroutineAllocator&; }; diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index 8234c5ebb7..1a08103090 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -2,118 +2,8 @@ // Copyright (c) 2024 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include -#include -#include -#include - #include "runtime-light/allocator/allocator-state.h" -#include "runtime-light/k2-platform/k2-api.h" -#include "runtime-light/stdlib/diagnostics/logs.h" RuntimeAllocator& RuntimeAllocator::get() noexcept { return AllocatorState::get_mutable().allocator; } - -RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) - : m_min_extra_mem_size(min_extra_mem_size) { - // kphp::log::debug("create runtime allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, - // oom_handling_mem_size); - void* buffer{alloc_global_memory(script_mem_size)}; - memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); -} - -void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) { - kphp::log::assertion(buffer != nullptr); - // kphp::log::debug("init runtime allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, - // script_mem_size, oom_handling_mem_size); - memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); -} - -void RuntimeAllocator::free() { - // kphp::log::debug("free runtime allocator -> {:p}", reinterpret_cast(this)); - auto* extra_memory{memory_resource.get_extra_memory_head()}; - while (extra_memory->get_pool_payload_size() != 0) { - auto* extra_memory_to_release{extra_memory}; - extra_memory = extra_memory->next_in_chain; - k2::free(extra_memory_to_release); - } - k2::free(memory_resource.memory_begin()); -} - -void* RuntimeAllocator::alloc_script_memory(size_t size) noexcept { - kphp::log::assertion(size != 0); - void* mem{memory_resource.allocate(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = memory_resource.allocate(size); - kphp::log::assertion(mem != nullptr); - } - return mem; -} - -void* RuntimeAllocator::alloc0_script_memory(size_t size) noexcept { - kphp::log::assertion(size != 0); - void* mem{memory_resource.allocate0(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = memory_resource.allocate0(size); - kphp::log::assertion(mem != nullptr); - } - return mem; -} - -void* RuntimeAllocator::realloc_script_memory(void* old_mem, size_t new_size, size_t old_size) noexcept { - kphp::log::assertion(new_size > old_size); - void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; - if (new_mem == nullptr) [[unlikely]] { - request_extra_memory(new_size * 2); - new_mem = memory_resource.reallocate(old_mem, new_size, old_size); - kphp::log::assertion(new_mem != nullptr); - } - return new_mem; -} - -void RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept { - kphp::log::assertion(size != 0); - memory_resource.deallocate(mem, size); -} - -void* RuntimeAllocator::alloc_global_memory(size_t size) noexcept { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - return mem; -} - -void* RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - std::memset(mem, 0, size); - return mem; -} - -void* RuntimeAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept { - void* mem{k2::realloc(old_mem, new_size)}; - kphp::log::assertion(mem != nullptr); - return mem; -} - -void RuntimeAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept { - k2::free(mem); -} - -void RuntimeAllocator::request_extra_memory(size_t requested_size) noexcept { - // Extra mem size have to be greater than max chunk block - const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; - - size_t extra_mem_size{std::max(min_size, requested_size)}; - // Take into account internal layout of `memory_resource::extra_memory_pool` - extra_mem_size += sizeof(memory_resource::extra_memory_pool); - // The smallest power of two that is not smaller than `extra_mem_size` - extra_mem_size = std::bit_ceil(extra_mem_size); - - // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); - - auto* extra_mem{alloc_global_memory(extra_mem_size)}; - memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); -} From 2eaa30394edcd2d95a617a060caf5233c5737767 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 17:40:07 +0300 Subject: [PATCH 19/30] add definitions of methods into runtime-allocator --- .../core/allocator/details/pool-allocator.h | 6 +-- .../core/allocator/runtime-allocator.h | 15 ++++++- runtime-light/allocator/coroutine-allocator.h | 1 - .../allocator/details/pool-allocator.cpp | 6 +-- .../allocator/runtime-light-allocator.cpp | 41 +++++++++++++++++++ runtime/context/runtime-core-allocator.cpp | 4 +- 6 files changed, 63 insertions(+), 10 deletions(-) diff --git a/runtime-common/core/allocator/details/pool-allocator.h b/runtime-common/core/allocator/details/pool-allocator.h index a11296e5ca..a75da0de85 100644 --- a/runtime-common/core/allocator/details/pool-allocator.h +++ b/runtime-common/core/allocator/details/pool-allocator.h @@ -13,10 +13,10 @@ namespace kphp::memory::details { struct PoolAllocator : vk::not_copyable { PoolAllocator() = default; - PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size); + PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; - auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) -> void; - auto free() -> void; + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; auto alloc_script_memory(size_t size) noexcept -> void*; auto alloc0_script_memory(size_t size) noexcept -> void*; diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index a121cdfae4..bc583fa3a9 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -9,7 +9,20 @@ #include "runtime-common/core/allocator/details/pool-allocator.h" struct RuntimeAllocator final : public kphp::memory::details::PoolAllocator { + static RuntimeAllocator& get() noexcept; + using kphp::memory::details::PoolAllocator::PoolAllocator; - static RuntimeAllocator& get() noexcept; + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; + + auto alloc_script_memory(size_t size) noexcept -> void*; + auto alloc0_script_memory(size_t size) noexcept -> void*; + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_script_memory(void* mem, size_t size) noexcept -> void; + + auto alloc_global_memory(size_t size) noexcept -> void*; + auto alloc0_global_memory(size_t size) noexcept -> void*; + auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_global_memory(void* mem, size_t size) noexcept -> void; }; diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h index 29e576653a..c0ea13c165 100644 --- a/runtime-light/allocator/coroutine-allocator.h +++ b/runtime-light/allocator/coroutine-allocator.h @@ -5,7 +5,6 @@ #pragma once #include -#include #include "runtime-light/allocator/runtime-coroutine-allocator.h" diff --git a/runtime-light/allocator/details/pool-allocator.cpp b/runtime-light/allocator/details/pool-allocator.cpp index cece64716c..5a17dd5683 100644 --- a/runtime-light/allocator/details/pool-allocator.cpp +++ b/runtime-light/allocator/details/pool-allocator.cpp @@ -13,7 +13,7 @@ namespace kphp::memory::details { -PoolAllocator::PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) +PoolAllocator::PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create pool allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, // oom_handling_mem_size); @@ -21,14 +21,14 @@ PoolAllocator::PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); } -auto PoolAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) -> void { +auto PoolAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { kphp::log::assertion(buffer != nullptr); // kphp::log::debug("init pool allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, // script_mem_size, oom_handling_mem_size); memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); } -auto PoolAllocator::free() -> void { +auto PoolAllocator::free() noexcept -> void { // kphp::log::debug("free pool allocator -> {:p}", reinterpret_cast(this)); auto* extra_memory{memory_resource.get_extra_memory_head()}; while (extra_memory->get_pool_payload_size() != 0) { diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index 1a08103090..4289ae7112 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -2,8 +2,49 @@ // Copyright (c) 2024 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt +#include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/allocator/allocator-state.h" RuntimeAllocator& RuntimeAllocator::get() noexcept { return AllocatorState::get_mutable().allocator; } + +auto RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + kphp::memory::details::PoolAllocator::PoolAllocator::init(buffer, script_mem_size, oom_handling_mem_size); +} + +auto RuntimeAllocator::free() noexcept -> void { + kphp::memory::details::PoolAllocator::PoolAllocator::free(); +} + +auto RuntimeAllocator::alloc_script_memory(size_t size) noexcept -> void* { + return kphp::memory::details::PoolAllocator::PoolAllocator::alloc_script_memory(size); +} + +auto RuntimeAllocator::alloc0_script_memory(size_t size) noexcept -> void* { + return kphp::memory::details::PoolAllocator::PoolAllocator::alloc0_script_memory(size); +} + +auto RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return kphp::memory::details::PoolAllocator::PoolAllocator::realloc_script_memory(mem, new_size, old_size); +} + +auto RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { + kphp::memory::details::PoolAllocator::PoolAllocator::free_script_memory(mem, size); +} + +auto RuntimeAllocator::alloc_global_memory(size_t size) noexcept -> void* { + return kphp::memory::details::PoolAllocator::PoolAllocator::alloc_global_memory(size); +} + +auto RuntimeAllocator::alloc0_global_memory(size_t size) noexcept -> void* { + return kphp::memory::details::PoolAllocator::PoolAllocator::alloc0_global_memory(size); +} + +auto RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return kphp::memory::details::PoolAllocator::PoolAllocator::realloc_global_memory(mem, new_size, old_size); +} + +auto RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { + kphp::memory::details::PoolAllocator::PoolAllocator::free_global_memory(mem, size); +} diff --git a/runtime/context/runtime-core-allocator.cpp b/runtime/context/runtime-core-allocator.cpp index 1d98c592a0..a2e9993760 100644 --- a/runtime/context/runtime-core-allocator.cpp +++ b/runtime/context/runtime-core-allocator.cpp @@ -5,11 +5,11 @@ #include "runtime/allocator.h" #include "runtime/context/runtime-context.h" -void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) { +void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept { dl::init_script_allocator(buffer, script_mem_size, oom_handling_mem_size); } -void RuntimeAllocator::free() { +void RuntimeAllocator::free() noexcept { dl::free_script_allocator(); } From 5ae2125e9d11eaea810f9a901067130b050b8444 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 17:57:08 +0300 Subject: [PATCH 20/30] replace inheritance with composition --- .../core/allocator/runtime-allocator.h | 11 +++-- .../allocator/runtime-coroutine-allocator.cpp | 44 +++++++++++++++++++ .../allocator/runtime-coroutine-allocator.h | 22 +++++++++- .../allocator/runtime-light-allocator.cpp | 25 ++++++----- 4 files changed, 86 insertions(+), 16 deletions(-) create mode 100644 runtime-light/allocator/runtime-coroutine-allocator.cpp diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index bc583fa3a9..6b7833be79 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -8,10 +8,15 @@ #include "runtime-common/core/allocator/details/pool-allocator.h" -struct RuntimeAllocator final : public kphp::memory::details::PoolAllocator { - static RuntimeAllocator& get() noexcept; +struct RuntimeAllocator final { +private: + kphp::memory::details::PoolAllocator m_allocator; - using kphp::memory::details::PoolAllocator::PoolAllocator; +public: + static auto get() noexcept -> RuntimeAllocator&; + + RuntimeAllocator() = default; + RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; auto free() noexcept -> void; diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp new file mode 100644 index 0000000000..58ed568243 --- /dev/null +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -0,0 +1,44 @@ +#include "runtime-light/allocator/runtime-coroutine-allocator.h" + +RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} + +auto RuntimeCoroutineAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + m_allocator.init(buffer, script_mem_size, oom_handling_mem_size); +} + +auto RuntimeCoroutineAllocator::free() noexcept -> void { + m_allocator.free(); +} + +auto RuntimeCoroutineAllocator::alloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc_script_memory(size); +} + +auto RuntimeCoroutineAllocator::alloc0_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc0_script_memory(size); +} + +auto RuntimeCoroutineAllocator::realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc_script_memory(mem, new_size, old_size); +} + +auto RuntimeCoroutineAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free_script_memory(mem, size); +} + +auto RuntimeCoroutineAllocator::alloc_global_memory(size_t size) noexcept -> void* { + return m_allocator.alloc_global_memory(size); +} + +auto RuntimeCoroutineAllocator::alloc0_global_memory(size_t size) noexcept -> void* { + return m_allocator.alloc0_global_memory(size); +} + +auto RuntimeCoroutineAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc_global_memory(mem, new_size, old_size); +} + +auto RuntimeCoroutineAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free_global_memory(mem, size); +} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index f45a59e704..8b2f5b8257 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -8,8 +8,26 @@ #include "runtime-common/core/allocator/details/pool-allocator.h" -struct RuntimeCoroutineAllocator final : public kphp::memory::details::PoolAllocator { - using kphp::memory::details::PoolAllocator::PoolAllocator; +struct RuntimeCoroutineAllocator final { +private: + kphp::memory::details::PoolAllocator m_allocator; +public: static auto get() noexcept -> RuntimeCoroutineAllocator&; + + RuntimeCoroutineAllocator() = default; + RuntimeCoroutineAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; + + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; + + auto alloc_script_memory(size_t size) noexcept -> void*; + auto alloc0_script_memory(size_t size) noexcept -> void*; + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_script_memory(void* mem, size_t size) noexcept -> void; + + auto alloc_global_memory(size_t size) noexcept -> void*; + auto alloc0_global_memory(size_t size) noexcept -> void*; + auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_global_memory(void* mem, size_t size) noexcept -> void; }; diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index 4289ae7112..fb649ef7a4 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -5,46 +5,49 @@ #include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/allocator/allocator-state.h" -RuntimeAllocator& RuntimeAllocator::get() noexcept { +auto RuntimeAllocator::get() noexcept -> RuntimeAllocator& { return AllocatorState::get_mutable().allocator; } +RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} + auto RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { - kphp::memory::details::PoolAllocator::PoolAllocator::init(buffer, script_mem_size, oom_handling_mem_size); + m_allocator.init(buffer, script_mem_size, oom_handling_mem_size); } auto RuntimeAllocator::free() noexcept -> void { - kphp::memory::details::PoolAllocator::PoolAllocator::free(); + m_allocator.free(); } auto RuntimeAllocator::alloc_script_memory(size_t size) noexcept -> void* { - return kphp::memory::details::PoolAllocator::PoolAllocator::alloc_script_memory(size); + return m_allocator.alloc_script_memory(size); } auto RuntimeAllocator::alloc0_script_memory(size_t size) noexcept -> void* { - return kphp::memory::details::PoolAllocator::PoolAllocator::alloc0_script_memory(size); + return m_allocator.alloc0_script_memory(size); } auto RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { - return kphp::memory::details::PoolAllocator::PoolAllocator::realloc_script_memory(mem, new_size, old_size); + return m_allocator.realloc_script_memory(mem, new_size, old_size); } auto RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { - kphp::memory::details::PoolAllocator::PoolAllocator::free_script_memory(mem, size); + m_allocator.free_script_memory(mem, size); } auto RuntimeAllocator::alloc_global_memory(size_t size) noexcept -> void* { - return kphp::memory::details::PoolAllocator::PoolAllocator::alloc_global_memory(size); + return m_allocator.alloc_global_memory(size); } auto RuntimeAllocator::alloc0_global_memory(size_t size) noexcept -> void* { - return kphp::memory::details::PoolAllocator::PoolAllocator::alloc0_global_memory(size); + return m_allocator.alloc0_global_memory(size); } auto RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { - return kphp::memory::details::PoolAllocator::PoolAllocator::realloc_global_memory(mem, new_size, old_size); + return m_allocator.realloc_global_memory(mem, new_size, old_size); } auto RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { - kphp::memory::details::PoolAllocator::PoolAllocator::free_global_memory(mem, size); + m_allocator.free_global_memory(mem, size); } From d5319398d3b18027e79d0226934e095a45732d14 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 18:08:08 +0300 Subject: [PATCH 21/30] add copyright header --- runtime-light/allocator/runtime-coroutine-allocator.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 58ed568243..2a496d166e 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -1,3 +1,7 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + #include "runtime-light/allocator/runtime-coroutine-allocator.h" RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept From ab8a7aeeb7894f74884e5386909b210126d2b8d5 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 18:23:36 +0300 Subject: [PATCH 22/30] add get_memory_resource method into runtime-allocator and runtime-coroutine-allocator --- runtime-common/core/allocator/runtime-allocator.h | 2 ++ .../allocator/runtime-coroutine-allocator.cpp | 4 ++++ runtime-light/allocator/runtime-coroutine-allocator.h | 2 ++ runtime-light/allocator/runtime-light-allocator.cpp | 4 ++++ runtime-light/stdlib/memory/memory-usage.h | 10 +++++----- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index 6b7833be79..fbec1c10da 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -30,4 +30,6 @@ struct RuntimeAllocator final { auto alloc0_global_memory(size_t size) noexcept -> void*; auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_global_memory(void* mem, size_t size) noexcept -> void; + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; }; diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 2a496d166e..7a5e409987 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -46,3 +46,7 @@ auto RuntimeCoroutineAllocator::realloc_global_memory(void* mem, size_t new_size auto RuntimeCoroutineAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { m_allocator.free_global_memory(mem, size); } + +auto RuntimeCoroutineAllocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.memory_resource; +} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index 8b2f5b8257..d372ac9599 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -30,4 +30,6 @@ struct RuntimeCoroutineAllocator final { auto alloc0_global_memory(size_t size) noexcept -> void*; auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_global_memory(void* mem, size_t size) noexcept -> void; + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; }; diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index fb649ef7a4..95595d7014 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -51,3 +51,7 @@ auto RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t auto RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { m_allocator.free_global_memory(mem, size); } + +auto RuntimeAllocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.memory_resource; +} diff --git a/runtime-light/stdlib/memory/memory-usage.h b/runtime-light/stdlib/memory/memory-usage.h index 53cd3b158f..2391c8fc10 100644 --- a/runtime-light/stdlib/memory/memory-usage.h +++ b/runtime-light/stdlib/memory/memory-usage.h @@ -12,22 +12,22 @@ inline int64_t f$memory_get_peak_usage(bool real_usage = false) noexcept { if (real_usage) { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().max_real_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().max_real_memory_used); } else { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().max_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().max_memory_used); } } inline int64_t f$memory_get_usage([[maybe_unused]] bool real_usage = false) noexcept { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().memory_used); } inline int64_t f$memory_get_total_usage() noexcept { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().real_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().real_memory_used); } inline array f$memory_get_detailed_stats() noexcept { - const auto& stats{RuntimeAllocator::get().memory_resource.get_memory_stats()}; + const auto& stats{RuntimeAllocator::get().get_memory_resource().get_memory_stats()}; return array({std::make_pair(string{"memory_limit"}, static_cast(stats.memory_limit)), std::make_pair(string{"real_memory_used"}, static_cast(stats.real_memory_used)), std::make_pair(string{"memory_used"}, static_cast(stats.memory_used)), From 3ea686ba2d9970aeb6a2125521efdb7ea1e86f40 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 13 Aug 2026 23:20:15 +0300 Subject: [PATCH 23/30] add sources to cmake --- runtime-light/allocator/allocator.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake index 85880d002e..7bba152586 100644 --- a/runtime-light/allocator/allocator.cmake +++ b/runtime-light/allocator/allocator.cmake @@ -1 +1,4 @@ -set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp) +set(RUNTIME_LIGHT_ALLOCATOR_SRC + allocator/runtime-light-allocator.cpp + allocator/runtime-coroutine-allocator.cpp + allocator/details/pool-allocator.cpp) From d5f7f2ad98ef4f3c0010f10378ba1fd2238cb57d Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 17 Aug 2026 16:12:43 +0300 Subject: [PATCH 24/30] revert replacing script allocator with coroutine allocator in stdlib files --- .../stdlib/component/inter-component-session/client.h | 4 ++-- runtime-light/stdlib/fork/fork-state.h | 4 ++-- runtime-light/stdlib/fork/wait-queue-state.h | 4 ++-- runtime-light/stdlib/rpc/rpc-client-state.h | 4 ++-- runtime-light/stdlib/rpc/rpc-queue-state.h | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index fbc9043c11..2319afef54 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -14,7 +14,7 @@ #include #include -#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/allocator/script-allocator.h" #include "runtime-light/coroutine/event.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/component/inter-component-session/details/function-wrapper.h" @@ -28,7 +28,7 @@ namespace kphp::component::inter_component_session { // The client for inter-component communication over a stream in a client-server manner class client final { using query_id_type = uint64_t; - using query2notifier_type = kphp::stl::map; + using query2notifier_type = kphp::stl::map; public: enum class response_readiness : uint8_t { pending, ready }; diff --git a/runtime-light/stdlib/fork/fork-state.h b/runtime-light/stdlib/fork/fork-state.h index 0bfeff2fb3..d992960dfa 100644 --- a/runtime-light/stdlib/fork/fork-state.h +++ b/runtime-light/stdlib/fork/fork-state.h @@ -12,7 +12,7 @@ #include "common/mixin/not_copyable.h" #include "runtime-common/core/std/containers.h" -#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/allocator/script-allocator.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/exception-types.h" @@ -40,7 +40,7 @@ struct ForkInstanceState final : private vk::not_copyable { int64_t next_fork_id{FORK_ID_INIT}; // type erased tasks that represent forks - kphp::stl::unordered_map forks; + kphp::stl::unordered_map forks; public: int64_t current_id{FORK_ID_INIT}; diff --git a/runtime-light/stdlib/fork/wait-queue-state.h b/runtime-light/stdlib/fork/wait-queue-state.h index fd3faacfe5..cb90a08d26 100644 --- a/runtime-light/stdlib/fork/wait-queue-state.h +++ b/runtime-light/stdlib/fork/wait-queue-state.h @@ -9,13 +9,13 @@ #include #include "runtime-common/core/std/containers.h" -#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/allocator/script-allocator.h" #include "runtime-light/coroutine/await-set.h" class WaitQueueInstanceState { static constexpr int64_t WAIT_QUEUE_ID_INIT = 0; int64_t m_next_wait_queue_id{WAIT_QUEUE_ID_INIT}; - kphp::stl::unordered_map, kphp::memory::coroutine_allocator> m_queues; + kphp::stl::unordered_map, kphp::memory::script_allocator> m_queues; public: WaitQueueInstanceState() noexcept = default; diff --git a/runtime-light/stdlib/rpc/rpc-client-state.h b/runtime-light/stdlib/rpc/rpc-client-state.h index 22cd9bb97c..2b19674e73 100644 --- a/runtime-light/stdlib/rpc/rpc-client-state.h +++ b/runtime-light/stdlib/rpc/rpc-client-state.h @@ -12,7 +12,7 @@ #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" -#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/allocator/script-allocator.h" #include "runtime-light/coroutine/await-set.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/stdlib/rpc/rpc-constants.h" @@ -24,7 +24,7 @@ struct RpcClientInstanceState final : private vk::not_copyable { CurrentTlQuery current_client_query{}; int64_t current_query_id{kphp::rpc::VALID_QUERY_ID_RANGE_START}; - kphp::stl::unordered_map>, kphp::memory::coroutine_allocator> response_awaiter_tasks; + kphp::stl::unordered_map>, kphp::memory::script_allocator> response_awaiter_tasks; kphp::stl::unordered_map, kphp::memory::script_allocator> response_fetcher_instances; kphp::stl::unordered_map, kphp::memory::script_allocator> rpc_responses_extra_info; diff --git a/runtime-light/stdlib/rpc/rpc-queue-state.h b/runtime-light/stdlib/rpc/rpc-queue-state.h index d7598696fa..3e712f154f 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-state.h +++ b/runtime-light/stdlib/rpc/rpc-queue-state.h @@ -10,13 +10,13 @@ #include "common/mixin/not_copyable.h" #include "runtime-common/core/std/containers.h" -#include "runtime-light/allocator/coroutine-allocator.h" +#include "runtime-light/allocator/script-allocator.h" #include "runtime-light/coroutine/await-set.h" class RpcQueueInstanceState final : private vk::not_copyable { static constexpr int64_t RPC_QUEUE_ID_INIT = 0; int64_t m_rpc_wait_queue_id{RPC_QUEUE_ID_INIT}; - kphp::stl::unordered_map, kphp::memory::coroutine_allocator> m_queues; + kphp::stl::unordered_map, kphp::memory::script_allocator> m_queues; public: RpcQueueInstanceState() noexcept = default; From 21a194322d86f996fb71b65d828f42033566bc64 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 17 Aug 2026 16:16:29 +0300 Subject: [PATCH 25/30] revert some more changes in stdlib files --- runtime-light/stdlib/component/inter-component-session/client.h | 1 - runtime-light/stdlib/fork/fork-state.h | 2 +- runtime-light/stdlib/fork/wait-queue-state.h | 2 +- runtime-light/stdlib/rpc/rpc-client-state.h | 1 - runtime-light/stdlib/rpc/rpc-queue-state.h | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 2319afef54..934b958ed9 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -14,7 +14,6 @@ #include #include -#include "runtime-light/allocator/script-allocator.h" #include "runtime-light/coroutine/event.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/component/inter-component-session/details/function-wrapper.h" diff --git a/runtime-light/stdlib/fork/fork-state.h b/runtime-light/stdlib/fork/fork-state.h index d992960dfa..0b14f9e284 100644 --- a/runtime-light/stdlib/fork/fork-state.h +++ b/runtime-light/stdlib/fork/fork-state.h @@ -11,8 +11,8 @@ #include #include "common/mixin/not_copyable.h" -#include "runtime-common/core/std/containers.h" #include "runtime-light/allocator/script-allocator.h" +#include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/exception-types.h" diff --git a/runtime-light/stdlib/fork/wait-queue-state.h b/runtime-light/stdlib/fork/wait-queue-state.h index cb90a08d26..932aa05c6e 100644 --- a/runtime-light/stdlib/fork/wait-queue-state.h +++ b/runtime-light/stdlib/fork/wait-queue-state.h @@ -8,8 +8,8 @@ #include #include -#include "runtime-common/core/std/containers.h" #include "runtime-light/allocator/script-allocator.h" +#include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/await-set.h" class WaitQueueInstanceState { diff --git a/runtime-light/stdlib/rpc/rpc-client-state.h b/runtime-light/stdlib/rpc/rpc-client-state.h index 2b19674e73..c4d86416b2 100644 --- a/runtime-light/stdlib/rpc/rpc-client-state.h +++ b/runtime-light/stdlib/rpc/rpc-client-state.h @@ -12,7 +12,6 @@ #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" -#include "runtime-light/allocator/script-allocator.h" #include "runtime-light/coroutine/await-set.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/stdlib/rpc/rpc-constants.h" diff --git a/runtime-light/stdlib/rpc/rpc-queue-state.h b/runtime-light/stdlib/rpc/rpc-queue-state.h index 3e712f154f..b333f79096 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-state.h +++ b/runtime-light/stdlib/rpc/rpc-queue-state.h @@ -9,8 +9,8 @@ #include #include "common/mixin/not_copyable.h" -#include "runtime-common/core/std/containers.h" #include "runtime-light/allocator/script-allocator.h" +#include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/await-set.h" class RpcQueueInstanceState final : private vk::not_copyable { From 082ce48e1a389b2ee6c9dbc92cfad4ec8803cf42 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 17 Aug 2026 16:21:13 +0300 Subject: [PATCH 26/30] revert some more changes in stdlib files --- runtime-light/stdlib/fork/fork-state.h | 2 +- runtime-light/stdlib/fork/wait-queue-state.h | 2 +- runtime-light/stdlib/rpc/rpc-queue-state.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime-light/stdlib/fork/fork-state.h b/runtime-light/stdlib/fork/fork-state.h index 0b14f9e284..d98eddccb5 100644 --- a/runtime-light/stdlib/fork/fork-state.h +++ b/runtime-light/stdlib/fork/fork-state.h @@ -11,7 +11,7 @@ #include #include "common/mixin/not_copyable.h" -#include "runtime-light/allocator/script-allocator.h" +#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" diff --git a/runtime-light/stdlib/fork/wait-queue-state.h b/runtime-light/stdlib/fork/wait-queue-state.h index 932aa05c6e..e9eac3e7a9 100644 --- a/runtime-light/stdlib/fork/wait-queue-state.h +++ b/runtime-light/stdlib/fork/wait-queue-state.h @@ -8,7 +8,7 @@ #include #include -#include "runtime-light/allocator/script-allocator.h" +#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/await-set.h" diff --git a/runtime-light/stdlib/rpc/rpc-queue-state.h b/runtime-light/stdlib/rpc/rpc-queue-state.h index b333f79096..7c307dd27c 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-state.h +++ b/runtime-light/stdlib/rpc/rpc-queue-state.h @@ -9,7 +9,7 @@ #include #include "common/mixin/not_copyable.h" -#include "runtime-light/allocator/script-allocator.h" +#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/await-set.h" From e108e7804c64c8521623f95ede3f592fd570e620 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 18 Aug 2026 17:58:20 +0300 Subject: [PATCH 27/30] small fixes --- .../core/allocator/details/malloc-interface.h | 22 ++---- .../core/allocator/details/pool-allocator.h | 8 +- .../core/allocator/runtime-allocator.h | 4 +- .../core/allocator/script-malloc-interface.h | 24 ++---- .../memory-resource/chunk-pool-resource.h | 74 +++++++++++++++++++ .../allocator/coroutine-malloc-interface.h | 24 ++---- .../allocator/details/pool-allocator.cpp | 28 ++++--- .../allocator/runtime-coroutine-allocator.cpp | 4 - .../allocator/runtime-coroutine-allocator.h | 4 +- .../allocator/runtime-light-allocator.cpp | 4 - .../confdata/state/component-state.h | 11 --- .../confdata/state/instance-state.h | 11 ++- 12 files changed, 127 insertions(+), 91 deletions(-) create mode 100644 runtime-common/core/memory-resource/chunk-pool-resource.h diff --git a/runtime-common/core/allocator/details/malloc-interface.h b/runtime-common/core/allocator/details/malloc-interface.h index ab491507ac..ea34faa96e 100644 --- a/runtime-common/core/allocator/details/malloc-interface.h +++ b/runtime-common/core/allocator/details/malloc-interface.h @@ -14,11 +14,7 @@ #include "common/wrappers/likely.h" #include "runtime-common/core/utils/kphp-assert-core.h" -namespace kphp { - -namespace memory { - -namespace details { +namespace kphp::memory::details { struct control_block { private: @@ -58,7 +54,7 @@ static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size m constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB -template +template struct malloc_interface { static auto alloc(size_t size) noexcept -> void* { constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; @@ -67,7 +63,7 @@ struct malloc_interface { return nullptr; } const size_t total_size{size + cb_size}; - void* base{AllocatorGetter().alloc_script_memory(total_size)}; + void* base{get_allocator_func().alloc_script_memory(total_size)}; if (unlikely(base == nullptr)) { php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); return base; @@ -94,7 +90,7 @@ struct malloc_interface { // Request mem from underlying memory manager const size_t total_size{size + (align - 1) + cb_size}; - void* base{AllocatorGetter().alloc_script_memory(total_size)}; + void* base{get_allocator_func().alloc_script_memory(total_size)}; if (unlikely(base == nullptr)) { php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); return base; @@ -131,7 +127,7 @@ struct malloc_interface { const auto cb{kphp::memory::details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - AllocatorGetter().free_script_memory(base, cb.size); + get_allocator_func().free_script_memory(base, cb.size); } static auto realloc(void* ptr, size_t new_size) noexcept -> void* { @@ -155,7 +151,7 @@ struct malloc_interface { void* new_ptr{alloc(new_size)}; if (likely(new_ptr != nullptr)) { std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - AllocatorGetter().free_script_memory(old_base, old_size); + get_allocator_func().free_script_memory(old_base, old_size); } return new_ptr; } @@ -166,8 +162,4 @@ struct malloc_interface { } }; -} // namespace details - -} // namespace memory - -} // namespace kphp +} // namespace kphp::memory::details diff --git a/runtime-common/core/allocator/details/pool-allocator.h b/runtime-common/core/allocator/details/pool-allocator.h index a75da0de85..32d2601d68 100644 --- a/runtime-common/core/allocator/details/pool-allocator.h +++ b/runtime-common/core/allocator/details/pool-allocator.h @@ -11,9 +11,9 @@ namespace kphp::memory::details { -struct PoolAllocator : vk::not_copyable { - PoolAllocator() = default; - PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; +struct pool_allocator : vk::not_copyable { + pool_allocator() = default; + pool_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; auto free() noexcept -> void; @@ -28,6 +28,8 @@ struct PoolAllocator : vk::not_copyable { auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_global_memory(void* mem, size_t size) noexcept -> void; + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; + private: auto request_extra_memory(size_t requested_size) noexcept -> void; diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index fbec1c10da..b7846f0291 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -10,7 +10,7 @@ struct RuntimeAllocator final { private: - kphp::memory::details::PoolAllocator m_allocator; + kphp::memory::details::pool_allocator m_allocator; public: static auto get() noexcept -> RuntimeAllocator&; @@ -30,6 +30,4 @@ struct RuntimeAllocator final { auto alloc0_global_memory(size_t size) noexcept -> void*; auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_global_memory(void* mem, size_t size) noexcept -> void; - - auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; }; diff --git a/runtime-common/core/allocator/script-malloc-interface.h b/runtime-common/core/allocator/script-malloc-interface.h index 944c88b03f..de6b3f1775 100644 --- a/runtime-common/core/allocator/script-malloc-interface.h +++ b/runtime-common/core/allocator/script-malloc-interface.h @@ -10,38 +10,30 @@ #include "runtime-common/core/allocator/details/malloc-interface.h" #include "runtime-common/core/allocator/runtime-allocator.h" -namespace kphp { - -namespace memory { - -namespace script { +namespace kphp::memory::script { inline auto alloc(size_t size) noexcept -> void* { - return details::malloc_interface::alloc(size); + return kphp::memory::details::malloc_interface::alloc(size); } inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { - return details::malloc_interface::alloc_aligned(size, alignment); + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); } inline auto calloc(size_t num, size_t size) noexcept -> void* { - return details::malloc_interface::calloc(num, size); + return kphp::memory::details::malloc_interface::calloc(num, size); } inline auto free(void* ptr) noexcept -> void { - details::malloc_interface::free(ptr); + kphp::memory::details::malloc_interface::free(ptr); } inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { - return details::malloc_interface::realloc(ptr, new_size); + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); } inline auto strdup(const char* str1) noexcept -> char* { - return details::malloc_interface::strdup(str1); + return kphp::memory::details::malloc_interface::strdup(str1); } -} // namespace script - -} // namespace memory - -} // namespace kphp +} // namespace kphp::memory::script diff --git a/runtime-common/core/memory-resource/chunk-pool-resource.h b/runtime-common/core/memory-resource/chunk-pool-resource.h new file mode 100644 index 0000000000..c6b995f4c2 --- /dev/null +++ b/runtime-common/core/memory-resource/chunk-pool-resource.h @@ -0,0 +1,74 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "common/mixin/not_copyable.h" +#include "common/wrappers/likely.h" +#include "runtime-common/core/memory-resource/memory_resource.h" +#include "runtime-common/core/utils/kphp-assert-core.h" +#include +#include + +namespace memory_resource { + +template +class chunk_pool_resource : private vk::not_copyable { + static_assert(ChunkSize > 0, "size of chunk must be greater than 0"); + + struct header { + std::byte* m_next{nullptr}; + }; + + static_assert(ChunkSize >= sizeof(header), "size of chunk too small for intrusive list header"); + + std::byte* m_head{nullptr}; + + auto init_buffer(void* buffer, size_t buffer_size) noexcept -> void { + std::byte* curr_chunk{static_cast(buffer)}; + std::byte* last_chunk{m_head + buffer_size - ChunkSize}; + while (curr_chunk != last_chunk) { + std::byte* next_chunk = curr_chunk + ChunkSize; + new (curr_chunk) header{next_chunk}; + curr_chunk = next_chunk; + } + + new (last_chunk) header{nullptr}; + } + +public: + auto init(void* buffer, size_t buffer_size) noexcept -> void { + php_assert(buffer_size <= memory_buffer_limit() && ChunkSize <= buffer_size && buffer_size % ChunkSize == 0); + + m_head = static_cast(buffer); + init_buffer(buffer, buffer_size); + } + + auto allocate() noexcept -> void* { + if (unlikely(m_head == nullptr)) { + return nullptr; + } + + void* allocated_chunk{m_head}; + m_head = reinterpret_cast(m_head)->m_next; + + return allocated_chunk; + } + + auto allocate0() noexcept -> void* { + void* allocated_chunk{allocate()}; + if (likely(allocated_chunk != nullptr)) { + memset(allocated_chunk, 0x00, ChunkSize); + } + + return allocated_chunk; + } + + auto deallocate(void* mem) noexcept -> void { + new (mem) header{m_head}; + m_head = reinterpret_cast(mem); + } +}; + +} // namespace memory_resource diff --git a/runtime-light/allocator/coroutine-malloc-interface.h b/runtime-light/allocator/coroutine-malloc-interface.h index 47b7561a9b..e34d39597c 100644 --- a/runtime-light/allocator/coroutine-malloc-interface.h +++ b/runtime-light/allocator/coroutine-malloc-interface.h @@ -10,38 +10,30 @@ #include "runtime-common/core/allocator/details/malloc-interface.h" #include "runtime-light/allocator/runtime-coroutine-allocator.h" -namespace kphp { - -namespace memory { - -namespace coro { +namespace kphp::memory::coro { inline auto alloc(size_t size) noexcept -> void* { - return details::malloc_interface::alloc(size); + return kphp::memory::details::malloc_interface::alloc(size); } inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { - return details::malloc_interface::alloc_aligned(size, alignment); + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); } inline auto calloc(size_t num, size_t size) noexcept -> void* { - return details::malloc_interface::calloc(num, size); + return kphp::memory::details::malloc_interface::calloc(num, size); } inline auto free(void* ptr) noexcept -> void { - details::malloc_interface::free(ptr); + kphp::memory::details::malloc_interface::free(ptr); } inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { - return details::malloc_interface::realloc(ptr, new_size); + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); } inline auto strdup(const char* str1) noexcept -> char* { - return details::malloc_interface::strdup(str1); + return kphp::memory::details::malloc_interface::strdup(str1); } -} // namespace coro - -} // namespace memory - -} // namespace kphp +} // namespace kphp::memory::coro diff --git a/runtime-light/allocator/details/pool-allocator.cpp b/runtime-light/allocator/details/pool-allocator.cpp index 5a17dd5683..8817c6616c 100644 --- a/runtime-light/allocator/details/pool-allocator.cpp +++ b/runtime-light/allocator/details/pool-allocator.cpp @@ -13,7 +13,7 @@ namespace kphp::memory::details { -PoolAllocator::PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept +pool_allocator::pool_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create pool allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, // oom_handling_mem_size); @@ -21,14 +21,14 @@ PoolAllocator::PoolAllocator(size_t script_mem_size, size_t min_extra_mem_size, memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); } -auto PoolAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { +auto pool_allocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { kphp::log::assertion(buffer != nullptr); // kphp::log::debug("init pool allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, // script_mem_size, oom_handling_mem_size); memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); } -auto PoolAllocator::free() noexcept -> void { +auto pool_allocator::free() noexcept -> void { // kphp::log::debug("free pool allocator -> {:p}", reinterpret_cast(this)); auto* extra_memory{memory_resource.get_extra_memory_head()}; while (extra_memory->get_pool_payload_size() != 0) { @@ -39,7 +39,7 @@ auto PoolAllocator::free() noexcept -> void { k2::free(memory_resource.memory_begin()); } -auto PoolAllocator::alloc_script_memory(size_t size) noexcept -> void* { +auto pool_allocator::alloc_script_memory(size_t size) noexcept -> void* { kphp::log::assertion(size != 0); void* mem{memory_resource.allocate(size)}; if (mem == nullptr) [[unlikely]] { @@ -50,7 +50,7 @@ auto PoolAllocator::alloc_script_memory(size_t size) noexcept -> void* { return mem; } -auto PoolAllocator::alloc0_script_memory(size_t size) noexcept -> void* { +auto pool_allocator::alloc0_script_memory(size_t size) noexcept -> void* { kphp::log::assertion(size != 0); void* mem{memory_resource.allocate0(size)}; if (mem == nullptr) [[unlikely]] { @@ -61,7 +61,7 @@ auto PoolAllocator::alloc0_script_memory(size_t size) noexcept -> void* { return mem; } -auto PoolAllocator::realloc_script_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { +auto pool_allocator::realloc_script_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { kphp::log::assertion(new_size > old_size); void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; if (new_mem == nullptr) [[unlikely]] { @@ -72,35 +72,35 @@ auto PoolAllocator::realloc_script_memory(void* old_mem, size_t new_size, size_t return new_mem; } -auto PoolAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { +auto pool_allocator::free_script_memory(void* mem, size_t size) noexcept -> void { kphp::log::assertion(size != 0); memory_resource.deallocate(mem, size); } -auto PoolAllocator::alloc_global_memory(size_t size) noexcept -> void* { +auto pool_allocator::alloc_global_memory(size_t size) noexcept -> void* { void* mem{k2::alloc(size)}; kphp::log::assertion(mem != nullptr); return mem; } -auto PoolAllocator::alloc0_global_memory(size_t size) noexcept -> void* { +auto pool_allocator::alloc0_global_memory(size_t size) noexcept -> void* { void* mem{k2::alloc(size)}; kphp::log::assertion(mem != nullptr); std::memset(mem, 0, size); return mem; } -auto PoolAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept -> void* { +auto pool_allocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept -> void* { void* mem{k2::realloc(old_mem, new_size)}; kphp::log::assertion(mem != nullptr); return mem; } -auto PoolAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept -> void { +auto pool_allocator::free_global_memory(void* mem, size_t /*unused*/) noexcept -> void { k2::free(mem); } -auto PoolAllocator::request_extra_memory(size_t requested_size) noexcept -> void { +auto pool_allocator::request_extra_memory(size_t requested_size) noexcept -> void { // Extra mem size have to be greater than max chunk block const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; @@ -116,4 +116,8 @@ auto PoolAllocator::request_extra_memory(size_t requested_size) noexcept -> void memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); } +auto pool_allocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return memory_resource; +} + } // namespace kphp::memory::details diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 7a5e409987..2a496d166e 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -46,7 +46,3 @@ auto RuntimeCoroutineAllocator::realloc_global_memory(void* mem, size_t new_size auto RuntimeCoroutineAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { m_allocator.free_global_memory(mem, size); } - -auto RuntimeCoroutineAllocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { - return m_allocator.memory_resource; -} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index d372ac9599..986e756c8f 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -10,7 +10,7 @@ struct RuntimeCoroutineAllocator final { private: - kphp::memory::details::PoolAllocator m_allocator; + kphp::memory::details::pool_allocator m_allocator; public: static auto get() noexcept -> RuntimeCoroutineAllocator&; @@ -30,6 +30,4 @@ struct RuntimeCoroutineAllocator final { auto alloc0_global_memory(size_t size) noexcept -> void*; auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_global_memory(void* mem, size_t size) noexcept -> void; - - auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; }; diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index 95595d7014..fb649ef7a4 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -51,7 +51,3 @@ auto RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t auto RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { m_allocator.free_global_memory(mem, size); } - -auto RuntimeAllocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { - return m_allocator.memory_resource; -} diff --git a/runtime-light/components/confdata/state/component-state.h b/runtime-light/components/confdata/state/component-state.h index a0612da5cd..c5e3059aa6 100644 --- a/runtime-light/components/confdata/state/component-state.h +++ b/runtime-light/components/confdata/state/component-state.h @@ -10,20 +10,9 @@ #include "runtime-light/k2-platform/k2-api.h" struct ComponentState final : private vk::not_copyable { - uint64_t initial_instance_memory_size{INIT_INSTANCE_ALLOCATOR_SIZE}; - uint64_t min_instance_extra_memory_size{DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE}; - uint64_t initial_instance_coroutine_memory_size{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; - uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; - ComponentState() noexcept = default; static auto get() noexcept -> const ComponentState&; static auto get_mutable() noexcept -> ComponentState&; - -private: - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB - static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(2U * 1024U * 1024U); // 2MiB - static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(512U * 1024U); // 0.5MiB }; inline auto ComponentState::get() noexcept -> const ComponentState& { diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 3530a5c6e4..2fa262050b 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -8,7 +8,6 @@ #include "common/mixin/not_copyable.h" #include "runtime-light/allocator/allocator-state.h" -#include "runtime-light/components/confdata/state/component-state.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/task.h" @@ -16,9 +15,8 @@ #include "runtime-light/stdlib/diagnostics/contextual-tags.h" struct InstanceState final : vk::not_copyable { - AllocatorState instance_allocator_state{ComponentState::get().initial_instance_memory_size, ComponentState::get().min_instance_extra_memory_size, 0}; - RuntimeCoroutineAllocator coroutine_allocator{ComponentState::get().initial_instance_coroutine_memory_size, - ComponentState::get().min_instance_extra_coroutine_memory_size, 0}; + AllocatorState instance_allocator_state{INIT_INSTANCE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; + RuntimeCoroutineAllocator coroutine_allocator{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE, 0}; kphp::log::contextual_tags instance_tags; @@ -31,6 +29,11 @@ struct InstanceState final : vk::not_copyable { auto init() noexcept -> void; private: + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(2U * 1024U * 1024U); // 2MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(512U * 1024U); // 0.5MiB + auto run() noexcept -> kphp::coro::task<>; }; From 434a76d85f6c1d2de1af7869f12129c78ff0fab6 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 18 Aug 2026 18:35:34 +0300 Subject: [PATCH 28/30] add global-memory-allocator --- .../core/allocator/details/pool-allocator.h | 7 +---- .../core/allocator/global-memory-allocator.h | 18 ++++++++++++ .../core/allocator/platform-allocator.h | 6 ++-- .../allocator/platform-malloc-interface.h | 8 ++--- .../core/allocator/runtime-allocator.h | 5 ---- .../core-types/definition/string_buffer.cpp | 8 ++--- .../core-types/definition/string_buffer.inl | 3 +- runtime-light/allocator/allocator.cmake | 3 +- .../allocator/details/pool-allocator.cpp | 28 ++---------------- .../allocator/global-memory-allocator.cpp | 26 +++++++++++++++++ .../allocator/runtime-coroutine-allocator.cpp | 16 ---------- .../allocator/runtime-coroutine-allocator.h | 5 ---- .../allocator/runtime-light-allocator.cpp | 16 ---------- .../components/confdata/bindings/bindings.cpp | 7 +++++ .../confdata/state/instance-state.h | 2 ++ .../components/kphp/bindings/bindings.cpp | 8 +++++ .../components/kphp/state/instance-state.h | 2 ++ .../core/globals/php-script-globals.cpp | 6 ++-- runtime/context/global-memory-allocator.cpp | 29 +++++++++++++++++++ runtime/context/runtime-context.cpp | 1 + runtime/context/runtime-context.h | 2 ++ runtime/context/runtime-core-allocator.cpp | 20 ------------- runtime/runtime.cmake | 1 + 23 files changed, 118 insertions(+), 109 deletions(-) create mode 100644 runtime-common/core/allocator/global-memory-allocator.h create mode 100644 runtime-light/allocator/global-memory-allocator.cpp create mode 100644 runtime/context/global-memory-allocator.cpp diff --git a/runtime-common/core/allocator/details/pool-allocator.h b/runtime-common/core/allocator/details/pool-allocator.h index 32d2601d68..238e673963 100644 --- a/runtime-common/core/allocator/details/pool-allocator.h +++ b/runtime-common/core/allocator/details/pool-allocator.h @@ -11,7 +11,7 @@ namespace kphp::memory::details { -struct pool_allocator : vk::not_copyable { +struct pool_allocator : private vk::not_copyable { pool_allocator() = default; pool_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; @@ -23,11 +23,6 @@ struct pool_allocator : vk::not_copyable { auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_script_memory(void* mem, size_t size) noexcept -> void; - auto alloc_global_memory(size_t size) noexcept -> void*; - auto alloc0_global_memory(size_t size) noexcept -> void*; - auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; - auto free_global_memory(void* mem, size_t size) noexcept -> void; - auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; private: diff --git a/runtime-common/core/allocator/global-memory-allocator.h b/runtime-common/core/allocator/global-memory-allocator.h new file mode 100644 index 0000000000..3c2fece8ed --- /dev/null +++ b/runtime-common/core/allocator/global-memory-allocator.h @@ -0,0 +1,18 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" + +struct GlobalMemoryAllocator final : private vk::not_copyable { + static auto get() noexcept -> GlobalMemoryAllocator&; + + auto alloc_global_memory(size_t size) noexcept -> void*; + auto alloc0_global_memory(size_t size) noexcept -> void*; + auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_global_memory(void* mem, size_t size) noexcept -> void; +}; diff --git a/runtime-common/core/allocator/platform-allocator.h b/runtime-common/core/allocator/platform-allocator.h index 6ae1134683..c1564b41e4 100644 --- a/runtime-common/core/allocator/platform-allocator.h +++ b/runtime-common/core/allocator/platform-allocator.h @@ -6,7 +6,7 @@ #include -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" namespace kphp::memory { @@ -25,11 +25,11 @@ struct platform_allocator { }; constexpr value_type* allocate(size_t n) noexcept { - return static_cast(RuntimeAllocator::get().alloc_global_memory(n * sizeof(T))); + return static_cast(GlobalMemoryAllocator::get().alloc_global_memory(n * sizeof(T))); } constexpr void deallocate(T* p, size_t n) noexcept { - RuntimeAllocator::get().free_global_memory(p, n * sizeof(T)); + GlobalMemoryAllocator::get().free_global_memory(p, n * sizeof(T)); } }; diff --git a/runtime-common/core/allocator/platform-malloc-interface.h b/runtime-common/core/allocator/platform-malloc-interface.h index 5afb94558a..d28276eeec 100644 --- a/runtime-common/core/allocator/platform-malloc-interface.h +++ b/runtime-common/core/allocator/platform-malloc-interface.h @@ -10,7 +10,7 @@ #include #include "common/wrappers/likely.h" -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/utils/kphp-assert-core.h" namespace kphp::memory::platform { @@ -24,7 +24,7 @@ inline void* alloc(size_t size) noexcept { return nullptr; } const size_t real_size{size + MALLOC_REPLACER_SIZE_OFFSET}; - void* ptr{RuntimeAllocator::get().alloc_global_memory(real_size)}; + void* ptr{GlobalMemoryAllocator::get().alloc_global_memory(real_size)}; if (unlikely(ptr == nullptr)) { php_warning("not enough platform memory to allocate: %lu", size); @@ -45,7 +45,7 @@ inline void* calloc(size_t num, size_t size) noexcept { inline void free(void* ptr) noexcept { if (likely(ptr != nullptr)) { void* real_ptr{static_cast(ptr) - MALLOC_REPLACER_SIZE_OFFSET}; - RuntimeAllocator::get().free_global_memory(real_ptr, *static_cast(real_ptr)); + GlobalMemoryAllocator::get().free_global_memory(real_ptr, *static_cast(real_ptr)); } } @@ -65,7 +65,7 @@ inline void* realloc(void* ptr, size_t new_size) noexcept { void* new_ptr{kphp::memory::platform::alloc(new_size)}; if (likely(new_ptr != nullptr)) { std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - RuntimeAllocator::get().free_global_memory(real_ptr, old_size); + GlobalMemoryAllocator::get().free_global_memory(real_ptr, old_size); } return new_ptr; } diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index b7846f0291..b9ce64af00 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -25,9 +25,4 @@ struct RuntimeAllocator final { auto alloc0_script_memory(size_t size) noexcept -> void*; auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_script_memory(void* mem, size_t size) noexcept -> void; - - auto alloc_global_memory(size_t size) noexcept -> void*; - auto alloc0_global_memory(size_t size) noexcept -> void*; - auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; - auto free_global_memory(void* mem, size_t size) noexcept -> void; }; diff --git a/runtime-common/core/core-types/definition/string_buffer.cpp b/runtime-common/core/core-types/definition/string_buffer.cpp index f88f6c21c9..271433db67 100644 --- a/runtime-common/core/core-types/definition/string_buffer.cpp +++ b/runtime-common/core/core-types/definition/string_buffer.cpp @@ -5,11 +5,11 @@ #include #include -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" string_buffer::string_buffer(string::size_type buffer_len) noexcept - : buffer_end(static_cast(RuntimeAllocator::get().alloc_global_memory(buffer_len))), + : buffer_end(static_cast(GlobalMemoryAllocator::get().alloc_global_memory(buffer_len))), buffer_begin(buffer_end), buffer_len(buffer_len) {} @@ -20,7 +20,7 @@ string_buffer::string_buffer(string_buffer&& other) noexcept string_buffer& string_buffer::operator=(string_buffer&& other) noexcept { if (this != std::addressof(other)) { - RuntimeAllocator::get().free_global_memory(buffer_begin, buffer_len); + GlobalMemoryAllocator::get().free_global_memory(buffer_begin, buffer_len); buffer_end = std::exchange(other.buffer_end, nullptr); buffer_begin = std::exchange(other.buffer_begin, nullptr); buffer_len = std::exchange(other.buffer_len, 0); @@ -29,5 +29,5 @@ string_buffer& string_buffer::operator=(string_buffer&& other) noexcept { } string_buffer::~string_buffer() noexcept { - RuntimeAllocator::get().free_global_memory(buffer_begin, buffer_len); + GlobalMemoryAllocator::get().free_global_memory(buffer_begin, buffer_len); } diff --git a/runtime-common/core/core-types/definition/string_buffer.inl b/runtime-common/core/core-types/definition/string_buffer.inl index f3497a8676..9316f18b39 100644 --- a/runtime-common/core/core-types/definition/string_buffer.inl +++ b/runtime-common/core/core-types/definition/string_buffer.inl @@ -1,6 +1,7 @@ #pragma once #include "common/algorithms/simd-int-to-string.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #ifndef INCLUDED_FROM_KPHP_CORE #error "this file must be included only from runtime-core.h" @@ -26,7 +27,7 @@ inline void string_buffer::resize(string::size_type new_buffer_len) noexcept { } string::size_type current_len = size(); - if (void* new_mem = RuntimeAllocator::get().realloc_global_memory(buffer_begin, new_buffer_len, buffer_len)) { + if (void* new_mem = GlobalMemoryAllocator::get().realloc_global_memory(buffer_begin, new_buffer_len, buffer_len)) { buffer_begin = static_cast(new_mem); buffer_len = new_buffer_len; buffer_end = buffer_begin + current_len; diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake index 7bba152586..004986b061 100644 --- a/runtime-light/allocator/allocator.cmake +++ b/runtime-light/allocator/allocator.cmake @@ -1,4 +1,5 @@ set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp allocator/runtime-coroutine-allocator.cpp - allocator/details/pool-allocator.cpp) + allocator/details/pool-allocator.cpp + allocator/global-memory-allocator.cpp) diff --git a/runtime-light/allocator/details/pool-allocator.cpp b/runtime-light/allocator/details/pool-allocator.cpp index 8817c6616c..bc687d8dc9 100644 --- a/runtime-light/allocator/details/pool-allocator.cpp +++ b/runtime-light/allocator/details/pool-allocator.cpp @@ -8,6 +8,7 @@ #include #include "runtime-common/core/allocator/details/pool-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -17,7 +18,7 @@ pool_allocator::pool_allocator(size_t script_mem_size, size_t min_extra_mem_size : m_min_extra_mem_size(min_extra_mem_size) { // kphp::log::debug("create pool allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, // oom_handling_mem_size); - void* buffer{alloc_global_memory(script_mem_size)}; + void* buffer{GlobalMemoryAllocator::get().alloc_global_memory(script_mem_size)}; memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); } @@ -77,29 +78,6 @@ auto pool_allocator::free_script_memory(void* mem, size_t size) noexcept -> void memory_resource.deallocate(mem, size); } -auto pool_allocator::alloc_global_memory(size_t size) noexcept -> void* { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - return mem; -} - -auto pool_allocator::alloc0_global_memory(size_t size) noexcept -> void* { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - std::memset(mem, 0, size); - return mem; -} - -auto pool_allocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept -> void* { - void* mem{k2::realloc(old_mem, new_size)}; - kphp::log::assertion(mem != nullptr); - return mem; -} - -auto pool_allocator::free_global_memory(void* mem, size_t /*unused*/) noexcept -> void { - k2::free(mem); -} - auto pool_allocator::request_extra_memory(size_t requested_size) noexcept -> void { // Extra mem size have to be greater than max chunk block const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; @@ -112,7 +90,7 @@ auto pool_allocator::request_extra_memory(size_t requested_size) noexcept -> voi // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); - auto* extra_mem{alloc_global_memory(extra_mem_size)}; + auto* extra_mem{GlobalMemoryAllocator::get().alloc_global_memory(extra_mem_size)}; memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); } diff --git a/runtime-light/allocator/global-memory-allocator.cpp b/runtime-light/allocator/global-memory-allocator.cpp new file mode 100644 index 0000000000..741b1e03a0 --- /dev/null +++ b/runtime-light/allocator/global-memory-allocator.cpp @@ -0,0 +1,26 @@ +#include "runtime-common/core/allocator/global-memory-allocator.h" +#include "runtime-light/k2-platform/k2-api.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +auto GlobalMemoryAllocator::alloc_global_memory(size_t size) noexcept -> void* { + void* mem{k2::alloc(size)}; + kphp::log::assertion(mem != nullptr); + return mem; +} + +auto GlobalMemoryAllocator::alloc0_global_memory(size_t size) noexcept -> void* { + void* mem{k2::alloc(size)}; + kphp::log::assertion(mem != nullptr); + std::memset(mem, 0, size); + return mem; +} + +auto GlobalMemoryAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept -> void* { + void* mem{k2::realloc(old_mem, new_size)}; + kphp::log::assertion(mem != nullptr); + return mem; +} + +auto GlobalMemoryAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept -> void { + k2::free(mem); +} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 2a496d166e..0aec992c92 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -30,19 +30,3 @@ auto RuntimeCoroutineAllocator::realloc_script_memory(void* mem, size_t new_size auto RuntimeCoroutineAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { m_allocator.free_script_memory(mem, size); } - -auto RuntimeCoroutineAllocator::alloc_global_memory(size_t size) noexcept -> void* { - return m_allocator.alloc_global_memory(size); -} - -auto RuntimeCoroutineAllocator::alloc0_global_memory(size_t size) noexcept -> void* { - return m_allocator.alloc0_global_memory(size); -} - -auto RuntimeCoroutineAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { - return m_allocator.realloc_global_memory(mem, new_size, old_size); -} - -auto RuntimeCoroutineAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { - m_allocator.free_global_memory(mem, size); -} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index 986e756c8f..2e58223e16 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -25,9 +25,4 @@ struct RuntimeCoroutineAllocator final { auto alloc0_script_memory(size_t size) noexcept -> void*; auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_script_memory(void* mem, size_t size) noexcept -> void; - - auto alloc_global_memory(size_t size) noexcept -> void*; - auto alloc0_global_memory(size_t size) noexcept -> void*; - auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; - auto free_global_memory(void* mem, size_t size) noexcept -> void; }; diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index fb649ef7a4..c61260ba88 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -35,19 +35,3 @@ auto RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t auto RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { m_allocator.free_script_memory(mem, size); } - -auto RuntimeAllocator::alloc_global_memory(size_t size) noexcept -> void* { - return m_allocator.alloc_global_memory(size); -} - -auto RuntimeAllocator::alloc0_global_memory(size_t size) noexcept -> void* { - return m_allocator.alloc0_global_memory(size); -} - -auto RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { - return m_allocator.realloc_global_memory(mem, new_size, old_size); -} - -auto RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { - m_allocator.free_global_memory(mem, size); -} diff --git a/runtime-light/components/confdata/bindings/bindings.cpp b/runtime-light/components/confdata/bindings/bindings.cpp index 688f40f23c..715bef5e22 100644 --- a/runtime-light/components/confdata/bindings/bindings.cpp +++ b/runtime-light/components/confdata/bindings/bindings.cpp @@ -39,6 +39,13 @@ auto contextual_tags::try_get() noexcept -> std::optional GlobalMemoryAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->global_memory_allocator; + } + kphp::log::error("can't find global memory allocator"); +} + auto AllocatorState::get() noexcept -> const AllocatorState& { if (k2::instance_state() != nullptr) [[likely]] { return InstanceState::get().instance_allocator_state; diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 2fa262050b..87875d806f 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -7,6 +7,7 @@ #include #include "common/mixin/not_copyable.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-light/allocator/allocator-state.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/coroutine/io-scheduler.h" @@ -15,6 +16,7 @@ #include "runtime-light/stdlib/diagnostics/contextual-tags.h" struct InstanceState final : vk::not_copyable { + GlobalMemoryAllocator global_memory_allocator; AllocatorState instance_allocator_state{INIT_INSTANCE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; RuntimeCoroutineAllocator coroutine_allocator{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE, 0}; diff --git a/runtime-light/components/kphp/bindings/bindings.cpp b/runtime-light/components/kphp/bindings/bindings.cpp index cafebdae58..0708f7ebbb 100644 --- a/runtime-light/components/kphp/bindings/bindings.cpp +++ b/runtime-light/components/kphp/bindings/bindings.cpp @@ -6,6 +6,7 @@ #include #include +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-light/allocator/allocator-state.h" #include "runtime-light/allocator/runtime-coroutine-allocator.h" @@ -66,6 +67,13 @@ auto contextual_tags::try_get() noexcept -> std::optional GlobalMemoryAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->global_memory_allocator; + } + kphp::log::error("can't find global memory allocator"); +} + auto AllocatorState::get() noexcept -> const AllocatorState& { if (const auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { return instance_state_ptr->instance_allocator_state; diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index c2979fa4d5..ae509299a0 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -8,6 +8,7 @@ #include #include "common/mixin/not_copyable.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" #include "runtime-light/allocator/allocator-state.h" @@ -86,6 +87,7 @@ struct InstanceState final : vk::not_copyable { return instance_kind_; } + GlobalMemoryAllocator global_memory_allocator; AllocatorState instance_allocator_state{ComponentState::get().initial_instance_memory_size, ComponentState::get().min_instance_extra_memory_size, 0}; RuntimeCoroutineAllocator coroutine_allocator{ComponentState::get().initial_instance_coroutine_memory_size, ComponentState::get().min_instance_extra_coroutine_memory_size, 0}; diff --git a/runtime-light/core/globals/php-script-globals.cpp b/runtime-light/core/globals/php-script-globals.cpp index 298b304de9..972e2fb7a4 100644 --- a/runtime-light/core/globals/php-script-globals.cpp +++ b/runtime-light/core/globals/php-script-globals.cpp @@ -5,18 +5,18 @@ #include "php-script-globals.h" #include "common/php-functions.h" -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-light/stdlib/diagnostics/logs.h" void PhpScriptMutableGlobals::once_alloc_linear_mem(unsigned int n_bytes) { kphp::log::assertion(g_linear_mem == nullptr); - g_linear_mem = static_cast(RuntimeAllocator::get().alloc0_global_memory(n_bytes)); + g_linear_mem = static_cast(GlobalMemoryAllocator::get().alloc0_global_memory(n_bytes)); } void PhpScriptMutableGlobals::once_alloc_linear_mem(const char* lib_name, unsigned int n_bytes) { int64_t key_lib_name{string_hash(lib_name, strlen(lib_name))}; kphp::log::assertion(libs_linear_mem.find(key_lib_name) == libs_linear_mem.end()); - libs_linear_mem[key_lib_name] = static_cast(RuntimeAllocator::get().alloc0_global_memory(n_bytes)); + libs_linear_mem[key_lib_name] = static_cast(GlobalMemoryAllocator::get().alloc0_global_memory(n_bytes)); } char* PhpScriptMutableGlobals::get_linear_mem(const char* lib_name) const { diff --git a/runtime/context/global-memory-allocator.cpp b/runtime/context/global-memory-allocator.cpp new file mode 100644 index 0000000000..a8d70d7eea --- /dev/null +++ b/runtime/context/global-memory-allocator.cpp @@ -0,0 +1,29 @@ +#include + +#include "runtime-common/core/allocator/global-memory-allocator.h" +#include "runtime/allocator.h" +#include "runtime/context/runtime-context.h" + +auto GlobalMemoryAllocator::get() noexcept -> GlobalMemoryAllocator& { + return global_memory_allocator; +} + +auto GlobalMemoryAllocator::alloc_global_memory(size_t size) noexcept -> void* { + return dl::heap_allocate(size); +} + +auto GlobalMemoryAllocator::alloc0_global_memory(size_t size) noexcept -> void* { + void* ptr = dl::heap_allocate(size); + if (ptr != nullptr) { + memset(ptr, 0, size); + } + return ptr; +} + +auto GlobalMemoryAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return dl::heap_reallocate(mem, new_size, old_size); +} + +auto GlobalMemoryAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { + dl::heap_deallocate(mem, size); +} diff --git a/runtime/context/runtime-context.cpp b/runtime/context/runtime-context.cpp index 4317f8f693..cae820ef98 100644 --- a/runtime/context/runtime-context.cpp +++ b/runtime/context/runtime-context.cpp @@ -8,6 +8,7 @@ #include "server/php-engine-vars.h" RuntimeContext kphp_runtime_context; +GlobalMemoryAllocator global_memory_allocator; RuntimeAllocator runtime_allocator; RuntimeContext& RuntimeContext::get() noexcept { diff --git a/runtime/context/runtime-context.h b/runtime/context/runtime-context.h index b498f8cf35..af7f455b32 100644 --- a/runtime/context/runtime-context.h +++ b/runtime/context/runtime-context.h @@ -4,7 +4,9 @@ #pragma once +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" extern RuntimeContext kphp_runtime_context; +extern GlobalMemoryAllocator global_memory_allocator; extern RuntimeAllocator runtime_allocator; diff --git a/runtime/context/runtime-core-allocator.cpp b/runtime/context/runtime-core-allocator.cpp index a2e9993760..3fc74f4f6a 100644 --- a/runtime/context/runtime-core-allocator.cpp +++ b/runtime/context/runtime-core-allocator.cpp @@ -32,23 +32,3 @@ void* RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t void RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept { dl::deallocate(mem, size); } - -void* RuntimeAllocator::alloc_global_memory(size_t size) noexcept { - return dl::heap_allocate(size); -} - -void* RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { - void* ptr = dl::heap_allocate(size); - if (ptr != nullptr) { - memset(ptr, 0, size); - } - return ptr; -} - -void* RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept { - return dl::heap_reallocate(mem, new_size, old_size); -} - -void RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept { - dl::heap_deallocate(mem, size); -} diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 9615c1dcd4..9fcb41afd8 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -58,6 +58,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ ${KPHP_RUNTIME_PDO_MYSQL_SOURCES} ${KPHP_RUNTIME_PDO_PGSQL_SOURCES} allocator.cpp + context/global-memory-allocator.cpp context/runtime-core-allocator.cpp context/runtime-context.cpp array_functions.cpp From cced678365affd7dc9041a86c11bdb8246d6ada0 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 18 Aug 2026 18:40:33 +0300 Subject: [PATCH 29/30] delete redundant file --- .../memory-resource/chunk-pool-resource.h | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100644 runtime-common/core/memory-resource/chunk-pool-resource.h diff --git a/runtime-common/core/memory-resource/chunk-pool-resource.h b/runtime-common/core/memory-resource/chunk-pool-resource.h deleted file mode 100644 index c6b995f4c2..0000000000 --- a/runtime-common/core/memory-resource/chunk-pool-resource.h +++ /dev/null @@ -1,74 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2026 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include "common/mixin/not_copyable.h" -#include "common/wrappers/likely.h" -#include "runtime-common/core/memory-resource/memory_resource.h" -#include "runtime-common/core/utils/kphp-assert-core.h" -#include -#include - -namespace memory_resource { - -template -class chunk_pool_resource : private vk::not_copyable { - static_assert(ChunkSize > 0, "size of chunk must be greater than 0"); - - struct header { - std::byte* m_next{nullptr}; - }; - - static_assert(ChunkSize >= sizeof(header), "size of chunk too small for intrusive list header"); - - std::byte* m_head{nullptr}; - - auto init_buffer(void* buffer, size_t buffer_size) noexcept -> void { - std::byte* curr_chunk{static_cast(buffer)}; - std::byte* last_chunk{m_head + buffer_size - ChunkSize}; - while (curr_chunk != last_chunk) { - std::byte* next_chunk = curr_chunk + ChunkSize; - new (curr_chunk) header{next_chunk}; - curr_chunk = next_chunk; - } - - new (last_chunk) header{nullptr}; - } - -public: - auto init(void* buffer, size_t buffer_size) noexcept -> void { - php_assert(buffer_size <= memory_buffer_limit() && ChunkSize <= buffer_size && buffer_size % ChunkSize == 0); - - m_head = static_cast(buffer); - init_buffer(buffer, buffer_size); - } - - auto allocate() noexcept -> void* { - if (unlikely(m_head == nullptr)) { - return nullptr; - } - - void* allocated_chunk{m_head}; - m_head = reinterpret_cast(m_head)->m_next; - - return allocated_chunk; - } - - auto allocate0() noexcept -> void* { - void* allocated_chunk{allocate()}; - if (likely(allocated_chunk != nullptr)) { - memset(allocated_chunk, 0x00, ChunkSize); - } - - return allocated_chunk; - } - - auto deallocate(void* mem) noexcept -> void { - new (mem) header{m_head}; - m_head = reinterpret_cast(mem); - } -}; - -} // namespace memory_resource From 3eec5f2dd0839a1089de7010b57a923c099672d3 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 18 Aug 2026 18:56:53 +0300 Subject: [PATCH 30/30] fix compilation error --- runtime-common/core/allocator/runtime-allocator.h | 4 ++++ runtime-light/allocator/runtime-coroutine-allocator.cpp | 4 ++++ runtime-light/allocator/runtime-coroutine-allocator.h | 2 ++ 3 files changed, 10 insertions(+) diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index b9ce64af00..f1c0985cbb 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -25,4 +25,8 @@ struct RuntimeAllocator final { auto alloc0_script_memory(size_t size) noexcept -> void*; auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_script_memory(void* mem, size_t size) noexcept -> void; + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.get_memory_resource(); + } }; diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp index 0aec992c92..db3ce94d6d 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.cpp +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -30,3 +30,7 @@ auto RuntimeCoroutineAllocator::realloc_script_memory(void* mem, size_t new_size auto RuntimeCoroutineAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { m_allocator.free_script_memory(mem, size); } + +auto RuntimeCoroutineAllocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.get_memory_resource(); +} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h index 2e58223e16..bab59d5a8b 100644 --- a/runtime-light/allocator/runtime-coroutine-allocator.h +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -25,4 +25,6 @@ struct RuntimeCoroutineAllocator final { auto alloc0_script_memory(size_t size) noexcept -> void*; auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; auto free_script_memory(void* mem, size_t size) noexcept -> void; + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; };