diff --git a/runtime-common/core/allocator/details/malloc-interface.h b/runtime-common/core/allocator/details/malloc-interface.h new file mode 100644 index 0000000000..ea34faa96e --- /dev/null +++ b/runtime-common/core/allocator/details/malloc-interface.h @@ -0,0 +1,165 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/wrappers/likely.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace kphp::memory::details { + +struct control_block { +private: + static constexpr auto SIZE_FIELD_BITSIZE{48}; + static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; + static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; + static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; + + static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); + +public: + static constexpr uint64_t max_size() noexcept { + return 1UL << SIZE_FIELD_BITSIZE; + } + + static constexpr uint64_t max_alignment() noexcept { + return 1UL << BASE_OFFSET_FIELD_BITSIZE; + } + + uint64_t raw() const noexcept { + return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); + } + + static control_block from_raw(uint64_t raw) noexcept { + return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; + } + + uint64_t size : SIZE_FIELD_BITSIZE; + uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; +}; + +inline bool is_power_of_2(uint64_t v) noexcept { + return v && !(v & (v - 1)); +} + +static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); + +constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB + +template +struct malloc_interface { + static auto alloc(size_t size) noexcept -> void* { + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + if (unlikely(size > std::min(kphp::memory::details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + const size_t total_size{size + cb_size}; + void* base{get_allocator_func().alloc_script_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + *(static_cast(base)) = kphp::memory::details::control_block{.size = total_size, .base_offset = cb_size}.raw(); + return static_cast(static_cast(base) + cb_size); + } + + static auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + // Check that provided alignment is power of two + const size_t align{static_cast(alignment)}; + if (unlikely(align == 0 || !kphp::memory::details::is_power_of_2(align) || align >= kphp::memory::details::control_block::max_alignment())) { + php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", + kphp::memory::details::control_block::max_alignment(), align); + return nullptr; + } + + // Check that memory is enough + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + if (unlikely(size > std::min(kphp::memory::details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + + // Request mem from underlying memory manager + const size_t total_size{size + (align - 1) + cb_size}; + void* base{get_allocator_func().alloc_script_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + + const uint64_t base_u{reinterpret_cast(base)}; + // The smallest multiple of `align` greater than or equal to requested memory + const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; + const uint64_t base_offset_u{aligned_u - base_u}; + + // Save control block + *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT + kphp::memory::details::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); + + return reinterpret_cast(aligned_u); // NOLINT + } + + static auto calloc(size_t num, size_t size) noexcept -> void* { + void* ptr{alloc(num * size)}; + if (unlikely(ptr == nullptr)) { + return nullptr; + } + return std::memset(ptr, 0, num * size); + } + + static auto free(void* ptr) noexcept -> void { + if (unlikely(ptr == nullptr)) { + return; + } + + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + + get_allocator_func().free_script_memory(base, cb.size); + } + + static auto realloc(void* ptr, size_t new_size) noexcept -> void* { + if (unlikely(ptr == nullptr)) { + return alloc(new_size); + } + + if (unlikely(new_size == 0)) { + free(ptr); + return nullptr; + } + + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + + void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + const size_t old_size{cb.size}; + + void* new_ptr{alloc(new_size)}; + if (likely(new_ptr != nullptr)) { + std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); + get_allocator_func().free_script_memory(old_base, old_size); + } + return new_ptr; + } + + static auto strdup(const char* str1) noexcept -> char* { + auto* str2{static_cast(alloc(std::strlen(str1) + 1))}; + return std::strcpy(str2, str1); + } +}; + +} // namespace kphp::memory::details diff --git a/runtime-common/core/allocator/details/pool-allocator.h b/runtime-common/core/allocator/details/pool-allocator.h new file mode 100644 index 0000000000..238e673963 --- /dev/null +++ b/runtime-common/core/allocator/details/pool-allocator.h @@ -0,0 +1,38 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" + +namespace kphp::memory::details { + +struct pool_allocator : private vk::not_copyable { + pool_allocator() = default; + pool_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; + + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; + + auto alloc_script_memory(size_t size) noexcept -> void*; + auto alloc0_script_memory(size_t size) noexcept -> void*; + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_script_memory(void* mem, size_t size) noexcept -> void; + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; + +private: + auto request_extra_memory(size_t requested_size) noexcept -> void; + +public: + memory_resource::unsynchronized_pool_resource memory_resource; + +private: + size_t m_min_extra_mem_size{0}; +}; + +} // namespace kphp::memory::details diff --git a/runtime-common/core/allocator/global-memory-allocator.h b/runtime-common/core/allocator/global-memory-allocator.h new file mode 100644 index 0000000000..3c2fece8ed --- /dev/null +++ b/runtime-common/core/allocator/global-memory-allocator.h @@ -0,0 +1,18 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" + +struct GlobalMemoryAllocator final : private vk::not_copyable { + static auto get() noexcept -> GlobalMemoryAllocator&; + + auto alloc_global_memory(size_t size) noexcept -> void*; + auto alloc0_global_memory(size_t size) noexcept -> void*; + auto realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_global_memory(void* mem, size_t size) noexcept -> void; +}; diff --git a/runtime-common/core/allocator/platform-allocator.h b/runtime-common/core/allocator/platform-allocator.h index 6ae1134683..c1564b41e4 100644 --- a/runtime-common/core/allocator/platform-allocator.h +++ b/runtime-common/core/allocator/platform-allocator.h @@ -6,7 +6,7 @@ #include -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" namespace kphp::memory { @@ -25,11 +25,11 @@ struct platform_allocator { }; constexpr value_type* allocate(size_t n) noexcept { - return static_cast(RuntimeAllocator::get().alloc_global_memory(n * sizeof(T))); + return static_cast(GlobalMemoryAllocator::get().alloc_global_memory(n * sizeof(T))); } constexpr void deallocate(T* p, size_t n) noexcept { - RuntimeAllocator::get().free_global_memory(p, n * sizeof(T)); + GlobalMemoryAllocator::get().free_global_memory(p, n * sizeof(T)); } }; diff --git a/runtime-common/core/allocator/platform-malloc-interface.h b/runtime-common/core/allocator/platform-malloc-interface.h index 5afb94558a..d28276eeec 100644 --- a/runtime-common/core/allocator/platform-malloc-interface.h +++ b/runtime-common/core/allocator/platform-malloc-interface.h @@ -10,7 +10,7 @@ #include #include "common/wrappers/likely.h" -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/utils/kphp-assert-core.h" namespace kphp::memory::platform { @@ -24,7 +24,7 @@ inline void* alloc(size_t size) noexcept { return nullptr; } const size_t real_size{size + MALLOC_REPLACER_SIZE_OFFSET}; - void* ptr{RuntimeAllocator::get().alloc_global_memory(real_size)}; + void* ptr{GlobalMemoryAllocator::get().alloc_global_memory(real_size)}; if (unlikely(ptr == nullptr)) { php_warning("not enough platform memory to allocate: %lu", size); @@ -45,7 +45,7 @@ inline void* calloc(size_t num, size_t size) noexcept { inline void free(void* ptr) noexcept { if (likely(ptr != nullptr)) { void* real_ptr{static_cast(ptr) - MALLOC_REPLACER_SIZE_OFFSET}; - RuntimeAllocator::get().free_global_memory(real_ptr, *static_cast(real_ptr)); + GlobalMemoryAllocator::get().free_global_memory(real_ptr, *static_cast(real_ptr)); } } @@ -65,7 +65,7 @@ inline void* realloc(void* ptr, size_t new_size) noexcept { void* new_ptr{kphp::memory::platform::alloc(new_size)}; if (likely(new_ptr != nullptr)) { std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - RuntimeAllocator::get().free_global_memory(real_ptr, old_size); + GlobalMemoryAllocator::get().free_global_memory(real_ptr, old_size); } return new_ptr; } diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index aac9d859d8..f1c0985cbb 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -6,34 +6,27 @@ #include -#include "common/mixin/not_copyable.h" -#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" +#include "runtime-common/core/allocator/details/pool-allocator.h" -struct RuntimeAllocator final : vk::not_copyable { - static RuntimeAllocator& get() noexcept; - - RuntimeAllocator() = default; - RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size); - - void init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size); - void free(); +struct RuntimeAllocator final { +private: + kphp::memory::details::pool_allocator m_allocator; - void* alloc_script_memory(size_t size) noexcept; - void* alloc0_script_memory(size_t size) noexcept; - void* realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept; - void free_script_memory(void* mem, size_t size) noexcept; +public: + static auto get() noexcept -> RuntimeAllocator&; - void* alloc_global_memory(size_t size) noexcept; - void* alloc0_global_memory(size_t size) noexcept; - void* realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept; - void free_global_memory(void* mem, size_t size) noexcept; + RuntimeAllocator() = default; + RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; -private: - void request_extra_memory(size_t requested_size) noexcept; + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; -public: - memory_resource::unsynchronized_pool_resource memory_resource; + auto alloc_script_memory(size_t size) noexcept -> void*; + auto alloc0_script_memory(size_t size) noexcept -> void*; + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_script_memory(void* mem, size_t size) noexcept -> void; -private: - size_t m_min_extra_mem_size{0}; + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.get_memory_resource(); + } }; diff --git a/runtime-common/core/allocator/script-malloc-interface.h b/runtime-common/core/allocator/script-malloc-interface.h index 6af4350f87..de6b3f1775 100644 --- a/runtime-common/core/allocator/script-malloc-interface.h +++ b/runtime-common/core/allocator/script-malloc-interface.h @@ -4,170 +4,36 @@ #pragma once -#include #include -#include #include -#include -#include "common/wrappers/likely.h" +#include "runtime-common/core/allocator/details/malloc-interface.h" #include "runtime-common/core/allocator/runtime-allocator.h" -#include "runtime-common/core/utils/kphp-assert-core.h" -namespace kphp { +namespace kphp::memory::script { -namespace memory { - -namespace script { - -constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB - -namespace details { -struct control_block { -private: - static constexpr auto SIZE_FIELD_BITSIZE{48}; - static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; - static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; - static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; - - static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); - -public: - static constexpr uint64_t max_size() noexcept { - return 1UL << SIZE_FIELD_BITSIZE; - } - - static constexpr uint64_t max_alignment() noexcept { - return 1UL << BASE_OFFSET_FIELD_BITSIZE; - } - - uint64_t raw() const noexcept { - return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); - } - - static control_block from_raw(uint64_t raw) noexcept { - return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; - } - - uint64_t size : SIZE_FIELD_BITSIZE; - uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; -}; - -inline bool is_power_of_2(uint64_t v) noexcept { - return v && !(v & (v - 1)); +inline auto alloc(size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc(size); } -static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); - -} // namespace details - -inline void* alloc(size_t size) noexcept { - constexpr size_t cb_size{sizeof(details::control_block)}; - if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - const size_t total_size{size + cb_size}; - void* base{RuntimeAllocator::get().alloc_script_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - *(static_cast(base)) = details::control_block{.size = total_size, .base_offset = cb_size}.raw(); - return static_cast(static_cast(base) + cb_size); +inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); } -inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { - // Check that provided alignment is power of two - const size_t align{static_cast(alignment)}; - if (unlikely(align == 0 || !details::is_power_of_2(align) || align >= details::control_block::max_alignment())) { - php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", details::control_block::max_alignment(), - align); - return nullptr; - } - - // Check that memory is enough - constexpr size_t cb_size{sizeof(details::control_block)}; - if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - - // Request mem from underlying memory manager - const size_t total_size{size + (align - 1) + cb_size}; - void* base{RuntimeAllocator::get().alloc_script_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - - const uint64_t base_u{reinterpret_cast(base)}; - // The smallest multiple of `align` greater than or equal to requested memory - const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; - const uint64_t base_offset_u{aligned_u - base_u}; - - // Save control block - *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT - details::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); - - return reinterpret_cast(aligned_u); // NOLINT -} - -inline void* calloc(size_t num, size_t size) noexcept { - void* ptr{kphp::memory::script::alloc(num * size)}; - if (unlikely(ptr == nullptr)) { - return nullptr; - } - return std::memset(ptr, 0, num * size); +inline auto calloc(size_t num, size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::calloc(num, size); } -inline void free(void* ptr) noexcept { - if (unlikely(ptr == nullptr)) { - return; - } - - constexpr size_t cb_size{sizeof(details::control_block)}; - const auto mem{reinterpret_cast(ptr)}; - - const auto cb{details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - - RuntimeAllocator::get().free_script_memory(base, cb.size); +inline auto free(void* ptr) noexcept -> void { + kphp::memory::details::malloc_interface::free(ptr); } -inline void* realloc(void* ptr, size_t new_size) noexcept { - if (unlikely(ptr == nullptr)) { - return kphp::memory::script::alloc(new_size); - } - - if (unlikely(new_size == 0)) { - kphp::memory::script::free(ptr); - return nullptr; - } - - constexpr size_t cb_size{sizeof(details::control_block)}; - const auto mem{reinterpret_cast(ptr)}; - - const auto cb{details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - - void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - const size_t old_size{cb.size}; - - void* new_ptr{kphp::memory::script::alloc(new_size)}; - if (likely(new_ptr != nullptr)) { - std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - RuntimeAllocator::get().free_script_memory(old_base, old_size); - } - return new_ptr; +inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); } -inline char* strdup(const char* str1) noexcept { - auto* str2{static_cast(kphp::memory::script::alloc(std::strlen(str1) + 1))}; - return std::strcpy(str2, str1); +inline auto strdup(const char* str1) noexcept -> char* { + return kphp::memory::details::malloc_interface::strdup(str1); } -} // namespace script - -} // namespace memory - -} // namespace kphp +} // namespace kphp::memory::script diff --git a/runtime-common/core/core-types/definition/string_buffer.cpp b/runtime-common/core/core-types/definition/string_buffer.cpp index f88f6c21c9..271433db67 100644 --- a/runtime-common/core/core-types/definition/string_buffer.cpp +++ b/runtime-common/core/core-types/definition/string_buffer.cpp @@ -5,11 +5,11 @@ #include #include -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" string_buffer::string_buffer(string::size_type buffer_len) noexcept - : buffer_end(static_cast(RuntimeAllocator::get().alloc_global_memory(buffer_len))), + : buffer_end(static_cast(GlobalMemoryAllocator::get().alloc_global_memory(buffer_len))), buffer_begin(buffer_end), buffer_len(buffer_len) {} @@ -20,7 +20,7 @@ string_buffer::string_buffer(string_buffer&& other) noexcept string_buffer& string_buffer::operator=(string_buffer&& other) noexcept { if (this != std::addressof(other)) { - RuntimeAllocator::get().free_global_memory(buffer_begin, buffer_len); + GlobalMemoryAllocator::get().free_global_memory(buffer_begin, buffer_len); buffer_end = std::exchange(other.buffer_end, nullptr); buffer_begin = std::exchange(other.buffer_begin, nullptr); buffer_len = std::exchange(other.buffer_len, 0); @@ -29,5 +29,5 @@ string_buffer& string_buffer::operator=(string_buffer&& other) noexcept { } string_buffer::~string_buffer() noexcept { - RuntimeAllocator::get().free_global_memory(buffer_begin, buffer_len); + GlobalMemoryAllocator::get().free_global_memory(buffer_begin, buffer_len); } diff --git a/runtime-common/core/core-types/definition/string_buffer.inl b/runtime-common/core/core-types/definition/string_buffer.inl index f3497a8676..9316f18b39 100644 --- a/runtime-common/core/core-types/definition/string_buffer.inl +++ b/runtime-common/core/core-types/definition/string_buffer.inl @@ -1,6 +1,7 @@ #pragma once #include "common/algorithms/simd-int-to-string.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #ifndef INCLUDED_FROM_KPHP_CORE #error "this file must be included only from runtime-core.h" @@ -26,7 +27,7 @@ inline void string_buffer::resize(string::size_type new_buffer_len) noexcept { } string::size_type current_len = size(); - if (void* new_mem = RuntimeAllocator::get().realloc_global_memory(buffer_begin, new_buffer_len, buffer_len)) { + if (void* new_mem = GlobalMemoryAllocator::get().realloc_global_memory(buffer_begin, new_buffer_len, buffer_len)) { buffer_begin = static_cast(new_mem); buffer_len = new_buffer_len; buffer_end = buffer_begin + current_len; diff --git a/runtime-light/allocator/allocator-state.h b/runtime-light/allocator/allocator-state.h index a5cc2e79e7..e5dde50264 100644 --- a/runtime-light/allocator/allocator-state.h +++ b/runtime-light/allocator/allocator-state.h @@ -11,8 +11,6 @@ #include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/stdlib/diagnostics/logs.h" -inline constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE{static_cast(1 * 1024U * 1024U)}; // 1Mib - class AllocatorState final : private vk::not_copyable { uint32_t m_libc_alloc_allowed{}; diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake index 85880d002e..004986b061 100644 --- a/runtime-light/allocator/allocator.cmake +++ b/runtime-light/allocator/allocator.cmake @@ -1 +1,5 @@ -set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp) +set(RUNTIME_LIGHT_ALLOCATOR_SRC + allocator/runtime-light-allocator.cpp + allocator/runtime-coroutine-allocator.cpp + allocator/details/pool-allocator.cpp + allocator/global-memory-allocator.cpp) diff --git a/runtime-light/allocator/coroutine-allocator.h b/runtime-light/allocator/coroutine-allocator.h new file mode 100644 index 0000000000..c0ea13c165 --- /dev/null +++ b/runtime-light/allocator/coroutine-allocator.h @@ -0,0 +1,45 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-light/allocator/runtime-coroutine-allocator.h" + +namespace kphp { + +namespace memory { + +template +struct coroutine_allocator { + using value_type = T; + + coroutine_allocator() noexcept = default; + + template + coroutine_allocator(const coroutine_allocator& /*unused*/) noexcept {} + + constexpr value_type* allocate(size_t n) noexcept { + return static_cast(RuntimeCoroutineAllocator::get().alloc_script_memory(n * sizeof(T))); + } + + constexpr void deallocate(T* p, size_t n) noexcept { + RuntimeCoroutineAllocator::get().free_script_memory(p, n * sizeof(T)); + } +}; + +template +constexpr bool operator==(const coroutine_allocator& /*unused*/, const coroutine_allocator& /*unused*/) { + return true; +} + +template +constexpr bool operator!=(const coroutine_allocator& /*unused*/, const coroutine_allocator& /*unused*/) { + return false; +} + +} // namespace memory + +} // namespace kphp diff --git a/runtime-light/allocator/coroutine-malloc-interface.h b/runtime-light/allocator/coroutine-malloc-interface.h new file mode 100644 index 0000000000..e34d39597c --- /dev/null +++ b/runtime-light/allocator/coroutine-malloc-interface.h @@ -0,0 +1,39 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-common/core/allocator/details/malloc-interface.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" + +namespace kphp::memory::coro { + +inline auto alloc(size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc(size); +} + +inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); +} + +inline auto calloc(size_t num, size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::calloc(num, size); +} + +inline auto free(void* ptr) noexcept -> void { + kphp::memory::details::malloc_interface::free(ptr); +} + +inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); +} + +inline auto strdup(const char* str1) noexcept -> char* { + return kphp::memory::details::malloc_interface::strdup(str1); +} + +} // namespace kphp::memory::coro diff --git a/runtime-light/allocator/details/pool-allocator.cpp b/runtime-light/allocator/details/pool-allocator.cpp new file mode 100644 index 0000000000..bc687d8dc9 --- /dev/null +++ b/runtime-light/allocator/details/pool-allocator.cpp @@ -0,0 +1,101 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include +#include + +#include "runtime-common/core/allocator/details/pool-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" +#include "runtime-light/k2-platform/k2-api.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::memory::details { + +pool_allocator::pool_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_min_extra_mem_size(min_extra_mem_size) { + // kphp::log::debug("create pool allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, + // oom_handling_mem_size); + void* buffer{GlobalMemoryAllocator::get().alloc_global_memory(script_mem_size)}; + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); +} + +auto pool_allocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + kphp::log::assertion(buffer != nullptr); + // kphp::log::debug("init pool allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, + // script_mem_size, oom_handling_mem_size); + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); +} + +auto pool_allocator::free() noexcept -> void { + // kphp::log::debug("free pool allocator -> {:p}", reinterpret_cast(this)); + auto* extra_memory{memory_resource.get_extra_memory_head()}; + while (extra_memory->get_pool_payload_size() != 0) { + auto* extra_memory_to_release{extra_memory}; + extra_memory = extra_memory->next_in_chain; + k2::free(extra_memory_to_release); + } + k2::free(memory_resource.memory_begin()); +} + +auto pool_allocator::alloc_script_memory(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + void* mem{memory_resource.allocate(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = memory_resource.allocate(size); + kphp::log::assertion(mem != nullptr); + } + return mem; +} + +auto pool_allocator::alloc0_script_memory(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + void* mem{memory_resource.allocate0(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = memory_resource.allocate0(size); + kphp::log::assertion(mem != nullptr); + } + return mem; +} + +auto pool_allocator::realloc_script_memory(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { + kphp::log::assertion(new_size > old_size); + void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; + if (new_mem == nullptr) [[unlikely]] { + request_extra_memory(new_size * 2); + new_mem = memory_resource.reallocate(old_mem, new_size, old_size); + kphp::log::assertion(new_mem != nullptr); + } + return new_mem; +} + +auto pool_allocator::free_script_memory(void* mem, size_t size) noexcept -> void { + kphp::log::assertion(size != 0); + memory_resource.deallocate(mem, size); +} + +auto pool_allocator::request_extra_memory(size_t requested_size) noexcept -> void { + // Extra mem size have to be greater than max chunk block + const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; + + size_t extra_mem_size{std::max(min_size, requested_size)}; + // Take into account internal layout of `memory_resource::extra_memory_pool` + extra_mem_size += sizeof(memory_resource::extra_memory_pool); + // The smallest power of two that is not smaller than `extra_mem_size` + extra_mem_size = std::bit_ceil(extra_mem_size); + + // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); + + auto* extra_mem{GlobalMemoryAllocator::get().alloc_global_memory(extra_mem_size)}; + memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); +} + +auto pool_allocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return memory_resource; +} + +} // namespace kphp::memory::details diff --git a/runtime-light/allocator/global-memory-allocator.cpp b/runtime-light/allocator/global-memory-allocator.cpp new file mode 100644 index 0000000000..741b1e03a0 --- /dev/null +++ b/runtime-light/allocator/global-memory-allocator.cpp @@ -0,0 +1,26 @@ +#include "runtime-common/core/allocator/global-memory-allocator.h" +#include "runtime-light/k2-platform/k2-api.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +auto GlobalMemoryAllocator::alloc_global_memory(size_t size) noexcept -> void* { + void* mem{k2::alloc(size)}; + kphp::log::assertion(mem != nullptr); + return mem; +} + +auto GlobalMemoryAllocator::alloc0_global_memory(size_t size) noexcept -> void* { + void* mem{k2::alloc(size)}; + kphp::log::assertion(mem != nullptr); + std::memset(mem, 0, size); + return mem; +} + +auto GlobalMemoryAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept -> void* { + void* mem{k2::realloc(old_mem, new_size)}; + kphp::log::assertion(mem != nullptr); + return mem; +} + +auto GlobalMemoryAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept -> void { + k2::free(mem); +} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.cpp b/runtime-light/allocator/runtime-coroutine-allocator.cpp new file mode 100644 index 0000000000..db3ce94d6d --- /dev/null +++ b/runtime-light/allocator/runtime-coroutine-allocator.cpp @@ -0,0 +1,36 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/allocator/runtime-coroutine-allocator.h" + +RuntimeCoroutineAllocator::RuntimeCoroutineAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} + +auto RuntimeCoroutineAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + m_allocator.init(buffer, script_mem_size, oom_handling_mem_size); +} + +auto RuntimeCoroutineAllocator::free() noexcept -> void { + m_allocator.free(); +} + +auto RuntimeCoroutineAllocator::alloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc_script_memory(size); +} + +auto RuntimeCoroutineAllocator::alloc0_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc0_script_memory(size); +} + +auto RuntimeCoroutineAllocator::realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc_script_memory(mem, new_size, old_size); +} + +auto RuntimeCoroutineAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free_script_memory(mem, size); +} + +auto RuntimeCoroutineAllocator::get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.get_memory_resource(); +} diff --git a/runtime-light/allocator/runtime-coroutine-allocator.h b/runtime-light/allocator/runtime-coroutine-allocator.h new file mode 100644 index 0000000000..bab59d5a8b --- /dev/null +++ b/runtime-light/allocator/runtime-coroutine-allocator.h @@ -0,0 +1,30 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-common/core/allocator/details/pool-allocator.h" + +struct RuntimeCoroutineAllocator final { +private: + kphp::memory::details::pool_allocator m_allocator; + +public: + static auto get() noexcept -> RuntimeCoroutineAllocator&; + + RuntimeCoroutineAllocator() = default; + RuntimeCoroutineAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; + + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; + + auto alloc_script_memory(size_t size) noexcept -> void*; + auto alloc0_script_memory(size_t size) noexcept -> void*; + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_script_memory(void* mem, size_t size) noexcept -> void; + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource&; +}; diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index 8234c5ebb7..c61260ba88 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -2,118 +2,36 @@ // Copyright (c) 2024 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include -#include -#include -#include - +#include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/allocator/allocator-state.h" -#include "runtime-light/k2-platform/k2-api.h" -#include "runtime-light/stdlib/diagnostics/logs.h" -RuntimeAllocator& RuntimeAllocator::get() noexcept { +auto RuntimeAllocator::get() noexcept -> RuntimeAllocator& { return AllocatorState::get_mutable().allocator; } -RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) - : m_min_extra_mem_size(min_extra_mem_size) { - // kphp::log::debug("create runtime allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, - // oom_handling_mem_size); - void* buffer{alloc_global_memory(script_mem_size)}; - memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); -} - -void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) { - kphp::log::assertion(buffer != nullptr); - // kphp::log::debug("init runtime allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, - // script_mem_size, oom_handling_mem_size); - memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); -} - -void RuntimeAllocator::free() { - // kphp::log::debug("free runtime allocator -> {:p}", reinterpret_cast(this)); - auto* extra_memory{memory_resource.get_extra_memory_head()}; - while (extra_memory->get_pool_payload_size() != 0) { - auto* extra_memory_to_release{extra_memory}; - extra_memory = extra_memory->next_in_chain; - k2::free(extra_memory_to_release); - } - k2::free(memory_resource.memory_begin()); -} - -void* RuntimeAllocator::alloc_script_memory(size_t size) noexcept { - kphp::log::assertion(size != 0); - void* mem{memory_resource.allocate(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = memory_resource.allocate(size); - kphp::log::assertion(mem != nullptr); - } - return mem; -} +RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} -void* RuntimeAllocator::alloc0_script_memory(size_t size) noexcept { - kphp::log::assertion(size != 0); - void* mem{memory_resource.allocate0(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = memory_resource.allocate0(size); - kphp::log::assertion(mem != nullptr); - } - return mem; +auto RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + m_allocator.init(buffer, script_mem_size, oom_handling_mem_size); } -void* RuntimeAllocator::realloc_script_memory(void* old_mem, size_t new_size, size_t old_size) noexcept { - kphp::log::assertion(new_size > old_size); - void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; - if (new_mem == nullptr) [[unlikely]] { - request_extra_memory(new_size * 2); - new_mem = memory_resource.reallocate(old_mem, new_size, old_size); - kphp::log::assertion(new_mem != nullptr); - } - return new_mem; +auto RuntimeAllocator::free() noexcept -> void { + m_allocator.free(); } -void RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept { - kphp::log::assertion(size != 0); - memory_resource.deallocate(mem, size); +auto RuntimeAllocator::alloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc_script_memory(size); } -void* RuntimeAllocator::alloc_global_memory(size_t size) noexcept { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - return mem; +auto RuntimeAllocator::alloc0_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc0_script_memory(size); } -void* RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - std::memset(mem, 0, size); - return mem; +auto RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc_script_memory(mem, new_size, old_size); } -void* RuntimeAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept { - void* mem{k2::realloc(old_mem, new_size)}; - kphp::log::assertion(mem != nullptr); - return mem; -} - -void RuntimeAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept { - k2::free(mem); -} - -void RuntimeAllocator::request_extra_memory(size_t requested_size) noexcept { - // Extra mem size have to be greater than max chunk block - const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; - - size_t extra_mem_size{std::max(min_size, requested_size)}; - // Take into account internal layout of `memory_resource::extra_memory_pool` - extra_mem_size += sizeof(memory_resource::extra_memory_pool); - // The smallest power of two that is not smaller than `extra_mem_size` - extra_mem_size = std::bit_ceil(extra_mem_size); - - // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); - - auto* extra_mem{alloc_global_memory(extra_mem_size)}; - memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); +auto RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free_script_memory(mem, size); } diff --git a/runtime-light/components/confdata/bindings/bindings.cpp b/runtime-light/components/confdata/bindings/bindings.cpp index 5d3ae4366a..715bef5e22 100644 --- a/runtime-light/components/confdata/bindings/bindings.cpp +++ b/runtime-light/components/confdata/bindings/bindings.cpp @@ -7,6 +7,7 @@ #include "runtime-common/core/runtime-core.h" #include "runtime-light/allocator/allocator-state.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/components/confdata/state/instance-state.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/coroutine/io-scheduler.h" @@ -38,6 +39,13 @@ auto contextual_tags::try_get() noexcept -> std::optional GlobalMemoryAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->global_memory_allocator; + } + kphp::log::error("can't find global memory allocator"); +} + auto AllocatorState::get() noexcept -> const AllocatorState& { if (k2::instance_state() != nullptr) [[likely]] { return InstanceState::get().instance_allocator_state; @@ -45,6 +53,13 @@ auto AllocatorState::get() noexcept -> const AllocatorState& { kphp::log::error("can't find allocator state"); } +auto RuntimeCoroutineAllocator::get() noexcept -> RuntimeCoroutineAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->coroutine_allocator; + } + kphp::log::error("can't find runtime coroutine allocator"); +} + auto ErrorHandlingState::try_get() noexcept -> std::optional> { return std::nullopt; // confdata doesn't support PHP error handling } diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 4a7243baea..87875d806f 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -7,6 +7,7 @@ #include #include "common/mixin/not_copyable.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-light/allocator/allocator-state.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/coroutine/io-scheduler.h" @@ -15,7 +16,9 @@ #include "runtime-light/stdlib/diagnostics/contextual-tags.h" struct InstanceState final : vk::not_copyable { + GlobalMemoryAllocator global_memory_allocator; AllocatorState instance_allocator_state{INIT_INSTANCE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; + RuntimeCoroutineAllocator coroutine_allocator{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE, 0}; kphp::log::contextual_tags instance_tags; @@ -28,7 +31,10 @@ struct InstanceState final : vk::not_copyable { auto init() noexcept -> void; private: - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(2U * 1024U * 1024U); // 2MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(512U * 1024U); // 0.5MiB auto run() noexcept -> kphp::coro::task<>; }; diff --git a/runtime-light/components/kphp/bindings/bindings.cpp b/runtime-light/components/kphp/bindings/bindings.cpp index 5f1b40b704..0708f7ebbb 100644 --- a/runtime-light/components/kphp/bindings/bindings.cpp +++ b/runtime-light/components/kphp/bindings/bindings.cpp @@ -6,8 +6,10 @@ #include #include +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-light/allocator/allocator-state.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/components/kphp/state/component-state.h" #include "runtime-light/components/kphp/state/image-state.h" #include "runtime-light/components/kphp/state/instance-state.h" @@ -65,6 +67,13 @@ auto contextual_tags::try_get() noexcept -> std::optional GlobalMemoryAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->global_memory_allocator; + } + kphp::log::error("can't find global memory allocator"); +} + auto AllocatorState::get() noexcept -> const AllocatorState& { if (const auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { return instance_state_ptr->instance_allocator_state; @@ -76,6 +85,13 @@ auto AllocatorState::get() noexcept -> const AllocatorState& { kphp::log::error("can't find allocator state"); } +auto RuntimeCoroutineAllocator::get() noexcept -> RuntimeCoroutineAllocator& { + if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { + return instance_state_ptr->coroutine_allocator; + } + kphp::log::error("can't find runtime coroutine allocator"); +} + auto RuntimeContext::get() noexcept -> RuntimeContext& { if (auto* instance_state_ptr{k2::instance_state()}; instance_state_ptr != nullptr) [[likely]] { return instance_state_ptr->runtime_context; diff --git a/runtime-light/components/kphp/state/component-state.cpp b/runtime-light/components/kphp/state/component-state.cpp index 9cd5b6e9da..251f73d35d 100644 --- a/runtime-light/components/kphp/state/component-state.cpp +++ b/runtime-light/components/kphp/state/component-state.cpp @@ -132,6 +132,24 @@ void ComponentState::parse_min_instance_extra_memory_size_arg(std::string_view v kphp::log::info("set min instance extra memory size to {} bytes", min_instance_extra_memory_size); } +void ComponentState::parse_initial_instance_coroutine_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse initial instance coroutine memory size, got {}", value_view); + } + initial_instance_coroutine_memory_size = *parsed; + kphp::log::info("set initial instance coroutine memory size to {} bytes", initial_instance_coroutine_memory_size); +} + +void ComponentState::parse_min_instance_extra_coroutine_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse min instance extra coroutine memory size, got {}", value_view); + } + min_instance_extra_coroutine_memory_size = *parsed; + kphp::log::info("set min instance extra coroutine memory size to {} bytes", min_instance_extra_coroutine_memory_size); +} + void ComponentState::parse_args() noexcept { for (auto i = 0; i < argc; ++i) { const auto [arg_key, arg_value]{k2::arg_fetch(i)}; @@ -152,6 +170,10 @@ void ComponentState::parse_args() noexcept { parse_initial_instance_memory_size_arg(value_view); } else if (key_view == MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG) { parse_min_instance_extra_memory_size_arg(value_view); + } else if (key_view == INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG) { + parse_initial_instance_coroutine_memory_size_arg(value_view); + } else if (key_view == MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG) { + parse_min_instance_extra_coroutine_memory_size_arg(value_view); } else { kphp::log::warning("unexpected argument format: {}", key_view); } diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index 9182b62107..0f72f4b418 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -31,6 +31,8 @@ struct ComponentState final : private vk::not_copyable { bool exit_after_response{}; uint64_t initial_instance_memory_size{INIT_INSTANCE_ALLOCATOR_SIZE}; uint64_t min_instance_extra_memory_size{DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE}; + uint64_t initial_instance_coroutine_memory_size{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; ComponentState() noexcept { parse_env(); @@ -63,8 +65,13 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view EXIT_AFTER_RESPONSE_ARG = "exit-after-response"; static constexpr std::string_view INITIAL_INSTANCE_MEMORY_SIZE_ARG = "initial-instance-memory-size"; static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG = "min-instance-extra-memory-size"; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr std::string_view INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG = "initial-instance-coroutine-memory-size"; + static constexpr std::string_view MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG = "min-instance-extra-coroutine-memory-size"; + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB + static constexpr auto DEFAULT_MIN_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB void parse_env() noexcept; @@ -83,4 +90,8 @@ struct ComponentState final : private vk::not_copyable { void parse_initial_instance_memory_size_arg(std::string_view) noexcept; void parse_min_instance_extra_memory_size_arg(std::string_view) noexcept; + + void parse_initial_instance_coroutine_memory_size_arg(std::string_view) noexcept; + + void parse_min_instance_extra_coroutine_memory_size_arg(std::string_view) noexcept; }; diff --git a/runtime-light/components/kphp/state/image-state.h b/runtime-light/components/kphp/state/image-state.h index 7d3fabae02..6245df78dd 100644 --- a/runtime-light/components/kphp/state/image-state.h +++ b/runtime-light/components/kphp/state/image-state.h @@ -102,5 +102,6 @@ struct ImageState final : private vk::not_copyable { } private: - static constexpr auto INIT_IMAGE_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_IMAGE_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB }; diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index 65ba170383..ae509299a0 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -8,9 +8,11 @@ #include #include "common/mixin/not_copyable.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" #include "runtime-light/allocator/allocator-state.h" +#include "runtime-light/allocator/runtime-coroutine-allocator.h" #include "runtime-light/components/kphp/state/component-state.h" #include "runtime-light/core/globals/php-script-globals.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -60,9 +62,6 @@ struct InstanceState final : vk::not_copyable { template using deque = kphp::stl::deque; - template - using list = kphp::stl::list; - // It's important to use `{}` instead of `= default` here. // In the second case clang++ zeroes the whole structure. // It drastically ruins performance. Be careful! @@ -88,7 +87,10 @@ struct InstanceState final : vk::not_copyable { return instance_kind_; } + GlobalMemoryAllocator global_memory_allocator; AllocatorState instance_allocator_state{ComponentState::get().initial_instance_memory_size, ComponentState::get().min_instance_extra_memory_size, 0}; + RuntimeCoroutineAllocator coroutine_allocator{ComponentState::get().initial_instance_coroutine_memory_size, + ComponentState::get().min_instance_extra_coroutine_memory_size, 0}; kphp::log::contextual_tags instance_tags; @@ -122,7 +124,7 @@ struct InstanceState final : vk::not_copyable { ErrorHandlingState error_handling_instance_state; KmlInstanceState kml_instance_state; - list> shutdown_functions; + kphp::stl::list, kphp::memory::coroutine_allocator> shutdown_functions; private: kphp::coro::task<> init_cli_instance() noexcept; diff --git a/runtime-light/core/globals/php-script-globals.cpp b/runtime-light/core/globals/php-script-globals.cpp index 298b304de9..972e2fb7a4 100644 --- a/runtime-light/core/globals/php-script-globals.cpp +++ b/runtime-light/core/globals/php-script-globals.cpp @@ -5,18 +5,18 @@ #include "php-script-globals.h" #include "common/php-functions.h" -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-light/stdlib/diagnostics/logs.h" void PhpScriptMutableGlobals::once_alloc_linear_mem(unsigned int n_bytes) { kphp::log::assertion(g_linear_mem == nullptr); - g_linear_mem = static_cast(RuntimeAllocator::get().alloc0_global_memory(n_bytes)); + g_linear_mem = static_cast(GlobalMemoryAllocator::get().alloc0_global_memory(n_bytes)); } void PhpScriptMutableGlobals::once_alloc_linear_mem(const char* lib_name, unsigned int n_bytes) { int64_t key_lib_name{string_hash(lib_name, strlen(lib_name))}; kphp::log::assertion(libs_linear_mem.find(key_lib_name) == libs_linear_mem.end()); - libs_linear_mem[key_lib_name] = static_cast(RuntimeAllocator::get().alloc0_global_memory(n_bytes)); + libs_linear_mem[key_lib_name] = static_cast(GlobalMemoryAllocator::get().alloc0_global_memory(n_bytes)); } char* PhpScriptMutableGlobals::get_linear_mem(const char* lib_name) const { diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 81fb98665e..97a81b27eb 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -11,7 +11,7 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -53,16 +53,16 @@ class await_broker { template void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } void start_task(await_set_task&& task, kphp::coro::async_stack_root& coroutine_stack_root, void* return_address) noexcept { @@ -175,16 +175,16 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { template void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } std::suspend_always initial_suspend() const noexcept { diff --git a/runtime-light/coroutine/detail/poll-info.h b/runtime-light/coroutine/detail/poll-info.h index 30c153a629..852670ae08 100644 --- a/runtime-light/coroutine/detail/poll-info.h +++ b/runtime-light/coroutine/detail/poll-info.h @@ -9,16 +9,16 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/poll.h" #include "runtime-light/k2-platform/k2-api.h" namespace kphp::coro::detail { struct poll_info { - using timed_events = kphp::stl::multimap; - using parked_polls = kphp::stl::multimap; + using timed_events = kphp::stl::multimap; + using parked_polls = kphp::stl::multimap; using scheduled_coroutines = vk::intrusive::list>>; // Each coroutine in the scheduler can be in one of the following states, represented by the `schedule_position` variant: diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index e6908f575b..c5e15fa5ed 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -9,7 +9,7 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -35,16 +35,16 @@ struct promise_self_deleting : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } auto get_return_object() noexcept -> task_self_deleting; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index 2386e4fbb6..da79154105 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -14,6 +14,7 @@ #include #include +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" @@ -152,16 +153,16 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } auto initial_suspend() const noexcept -> std::suspend_always { diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index f5ee01549e..588060f237 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -13,6 +13,7 @@ #include #include +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -162,16 +163,16 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } auto initial_suspend() const noexcept -> std::suspend_always { diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 4be3984ac8..5eec557c4d 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -13,7 +13,7 @@ #include "common/containers/intrusive-list.h" #include "common/mixin/not_copyable.h" #include "common/wrappers/overloaded.h" -#include "runtime-common/core/allocator/script-allocator-managed.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/coroutine-state.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -21,12 +21,24 @@ namespace kphp::coro { class event { - struct event_controller : kphp::memory::script_allocator_managed, vk::not_copyable { + struct event_controller : vk::not_copyable { // 1) std::monostate => not set and no coroutines are waiting // 2) non empty list => linked list of coroutines waiting for the event to trigger // 3) empty list => the event is triggered and all coroutines are resumed std::variant>>> m_state; + void* operator new(size_t n) noexcept { + return kphp::memory::coro::alloc(n); + } + + auto operator new(size_t n, std::align_val_t al) noexcept -> void* { + return kphp::memory::coro::alloc_aligned(n, al); + } + + void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { + kphp::memory::coro::free(ptr); + } + auto set() noexcept -> void; auto unset() noexcept -> void; auto is_set() const noexcept -> bool; diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 2cc9f01cf8..3579a8ba78 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -22,8 +22,8 @@ #include "common/containers/final_action.h" #include "common/containers/intrusive-list.h" #include "common/wrappers/overloaded.h" -#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/allocator/coroutine-allocator.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -49,7 +49,7 @@ class io_scheduler { kphp::coro::detail::timer_handle m_timer_handle; kphp::coro::detail::poll_info::timed_events m_timed_events; - kphp::stl::vector m_accepted_descriptors; + kphp::stl::vector m_accepted_descriptors; kphp::coro::detail::poll_info::parked_polls m_parked_polls; kphp::coro::detail::poll_info::scheduled_coroutines m_scheduled_coroutines; diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index af6fd6b591..f2958f6b81 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -16,7 +16,7 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -125,16 +125,16 @@ struct promise_base : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } private: diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index d5c064720b..f9a78fd177 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -11,7 +11,7 @@ #include #include "common/containers/final_action.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -66,16 +66,16 @@ struct promise_base : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::memory::coro::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::memory::coro::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::memory::coro::free(ptr); } void* m_next{}; diff --git a/runtime-light/stdlib/memory/memory-usage.h b/runtime-light/stdlib/memory/memory-usage.h index 53cd3b158f..2391c8fc10 100644 --- a/runtime-light/stdlib/memory/memory-usage.h +++ b/runtime-light/stdlib/memory/memory-usage.h @@ -12,22 +12,22 @@ inline int64_t f$memory_get_peak_usage(bool real_usage = false) noexcept { if (real_usage) { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().max_real_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().max_real_memory_used); } else { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().max_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().max_memory_used); } } inline int64_t f$memory_get_usage([[maybe_unused]] bool real_usage = false) noexcept { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().memory_used); } inline int64_t f$memory_get_total_usage() noexcept { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().real_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().real_memory_used); } inline array f$memory_get_detailed_stats() noexcept { - const auto& stats{RuntimeAllocator::get().memory_resource.get_memory_stats()}; + const auto& stats{RuntimeAllocator::get().get_memory_resource().get_memory_stats()}; return array({std::make_pair(string{"memory_limit"}, static_cast(stats.memory_limit)), std::make_pair(string{"real_memory_used"}, static_cast(stats.real_memory_used)), std::make_pair(string{"memory_used"}, static_cast(stats.memory_used)), diff --git a/runtime/context/global-memory-allocator.cpp b/runtime/context/global-memory-allocator.cpp new file mode 100644 index 0000000000..a8d70d7eea --- /dev/null +++ b/runtime/context/global-memory-allocator.cpp @@ -0,0 +1,29 @@ +#include + +#include "runtime-common/core/allocator/global-memory-allocator.h" +#include "runtime/allocator.h" +#include "runtime/context/runtime-context.h" + +auto GlobalMemoryAllocator::get() noexcept -> GlobalMemoryAllocator& { + return global_memory_allocator; +} + +auto GlobalMemoryAllocator::alloc_global_memory(size_t size) noexcept -> void* { + return dl::heap_allocate(size); +} + +auto GlobalMemoryAllocator::alloc0_global_memory(size_t size) noexcept -> void* { + void* ptr = dl::heap_allocate(size); + if (ptr != nullptr) { + memset(ptr, 0, size); + } + return ptr; +} + +auto GlobalMemoryAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return dl::heap_reallocate(mem, new_size, old_size); +} + +auto GlobalMemoryAllocator::free_global_memory(void* mem, size_t size) noexcept -> void { + dl::heap_deallocate(mem, size); +} diff --git a/runtime/context/runtime-context.cpp b/runtime/context/runtime-context.cpp index 4317f8f693..cae820ef98 100644 --- a/runtime/context/runtime-context.cpp +++ b/runtime/context/runtime-context.cpp @@ -8,6 +8,7 @@ #include "server/php-engine-vars.h" RuntimeContext kphp_runtime_context; +GlobalMemoryAllocator global_memory_allocator; RuntimeAllocator runtime_allocator; RuntimeContext& RuntimeContext::get() noexcept { diff --git a/runtime/context/runtime-context.h b/runtime/context/runtime-context.h index b498f8cf35..af7f455b32 100644 --- a/runtime/context/runtime-context.h +++ b/runtime/context/runtime-context.h @@ -4,7 +4,9 @@ #pragma once +#include "runtime-common/core/allocator/global-memory-allocator.h" #include "runtime-common/core/runtime-core.h" extern RuntimeContext kphp_runtime_context; +extern GlobalMemoryAllocator global_memory_allocator; extern RuntimeAllocator runtime_allocator; diff --git a/runtime/context/runtime-core-allocator.cpp b/runtime/context/runtime-core-allocator.cpp index 1d98c592a0..3fc74f4f6a 100644 --- a/runtime/context/runtime-core-allocator.cpp +++ b/runtime/context/runtime-core-allocator.cpp @@ -5,11 +5,11 @@ #include "runtime/allocator.h" #include "runtime/context/runtime-context.h" -void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) { +void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept { dl::init_script_allocator(buffer, script_mem_size, oom_handling_mem_size); } -void RuntimeAllocator::free() { +void RuntimeAllocator::free() noexcept { dl::free_script_allocator(); } @@ -32,23 +32,3 @@ void* RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t void RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept { dl::deallocate(mem, size); } - -void* RuntimeAllocator::alloc_global_memory(size_t size) noexcept { - return dl::heap_allocate(size); -} - -void* RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { - void* ptr = dl::heap_allocate(size); - if (ptr != nullptr) { - memset(ptr, 0, size); - } - return ptr; -} - -void* RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept { - return dl::heap_reallocate(mem, new_size, old_size); -} - -void RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept { - dl::heap_deallocate(mem, size); -} diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 9615c1dcd4..9fcb41afd8 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -58,6 +58,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ ${KPHP_RUNTIME_PDO_MYSQL_SOURCES} ${KPHP_RUNTIME_PDO_PGSQL_SOURCES} allocator.cpp + context/global-memory-allocator.cpp context/runtime-core-allocator.cpp context/runtime-context.cpp array_functions.cpp