diff --git a/opencl/extensions/public/cl_ext_private.h b/opencl/extensions/public/cl_ext_private.h index 7ee6d576b40a3..0a3f839315f4b 100644 --- a/opencl/extensions/public/cl_ext_private.h +++ b/opencl/extensions/public/cl_ext_private.h @@ -421,4 +421,11 @@ typedef struct _cl_kernel_allocation_info_intel { // cl_device bfloat16 atomic capabilities #if !defined(CL_DEVICE_BFLOAT16_FP_ATOMIC_CAPABILITIES_EXT) #define CL_DEVICE_BFLOAT16_FP_ATOMIC_CAPABILITIES_EXT 0x10012 + +/****************************************************** + * cl_intel_bindless_images extension * + ******************************************************/ +#define CL_MEM_BINDLESS_IMAGE_INTEL 0x10060 +#define CL_IMAGE_BINDLESS_HANDLE_INTEL 0x10061 + #endif diff --git a/opencl/source/helpers/api_specific_config_ocl.cpp b/opencl/source/helpers/api_specific_config_ocl.cpp index 495dcd7cb4a42..f1f38ed80f844 100644 --- a/opencl/source/helpers/api_specific_config_ocl.cpp +++ b/opencl/source/helpers/api_specific_config_ocl.cpp @@ -9,6 +9,7 @@ #include "shared/source/device/device.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/compiler_product_helper.h" +#include "shared/source/release_helper/release_helper.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -20,19 +21,20 @@ StackVec validClPrefixes; StackVec validClPrefixTypes; bool ApiSpecificConfig::getGlobalBindlessHeapConfiguration(const ReleaseHelper &releaseHelper) { - return false; + if (debugManager.flags.UseExternalAllocatorForSshAndDsh.get() != -1) { + return debugManager.flags.UseExternalAllocatorForSshAndDsh.get(); + } + return releaseHelper.isGlobalBindlessAllocatorEnabled(); } bool ApiSpecificConfig::getBindlessMode(const Device &device) { if (device.getCompilerProductHelper().isForceBindlessRequired(device.getHardwareInfo())) { return true; } - if (debugManager.flags.UseBindlessMode.get() != -1) { return debugManager.flags.UseBindlessMode.get(); - } else { - return false; } + return false; } bool ApiSpecificConfig::isDeviceAllocationCacheEnabled() { diff --git a/opencl/source/helpers/cl_memory_properties_helpers.cpp b/opencl/source/helpers/cl_memory_properties_helpers.cpp index 68843602c6564..61f97e36d1924 100644 --- a/opencl/source/helpers/cl_memory_properties_helpers.cpp +++ b/opencl/source/helpers/cl_memory_properties_helpers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2024 Intel Corporation + * Copyright (C) 2021-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int uint64_t handleType = 0; uintptr_t hostptr = 0; std::vector devices; - + bool bindlessImage = false; if (properties != nullptr) { for (int i = 0; properties[i] != 0; i += 2) { switch (properties[i]) { @@ -73,6 +73,9 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int i++; } break; + case CL_MEM_BINDLESS_IMAGE_INTEL: // Avoiding get a false when using bindless image extension + bindlessImage = true; + break; default: return false; } @@ -84,7 +87,7 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int memoryProperties.handle = handle; memoryProperties.hostptr = hostptr; memoryProperties.associatedDevices = devices; - + memoryProperties.flags.bindlessImage = bindlessImage; switch (objectType) { case ClMemoryPropertiesHelper::ObjType::buffer: return isFieldValid(flags, MemObjHelper::validFlagsForBuffer) && diff --git a/opencl/source/kernel/kernel.cpp b/opencl/source/kernel/kernel.cpp index 238089c2ae8bf..6e7c799129867 100644 --- a/opencl/source/kernel/kernel.cpp +++ b/opencl/source/kernel/kernel.cpp @@ -29,6 +29,7 @@ #include "shared/source/helpers/ptr_math.h" #include "shared/source/helpers/simd_helper.h" #include "shared/source/helpers/surface_format_info.h" +#include "shared/source/indirect_heap/indirect_heap.h" #include "shared/source/kernel/local_ids_cache.h" #include "shared/source/memory_manager/compression_selector.h" #include "shared/source/memory_manager/memory_manager.h" @@ -1334,7 +1335,17 @@ void Kernel::makeResident(CommandStreamReceiver &commandStreamReceiver) { pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(this->getContext().getSVMAllocsManager()); } makeArgsResident(commandStreamReceiver); - + auto bindlessHelper = getDevice().getDevice().getBindlessHeapsHelper(); + if (bindlessHelper) { + for (auto heapType : {NEO::BindlessHeapsHelper::specialSsh, + NEO::BindlessHeapsHelper::globalSsh, + NEO::BindlessHeapsHelper::globalDsh}) { + auto heap = bindlessHelper->getHeap(heapType); + if (heap) { + commandStreamReceiver.makeResident(*heap->getGraphicsAllocation()); + } + } + } auto kernelIsaAllocation = this->kernelInfo.getIsaGraphicsAllocation(); if (kernelIsaAllocation) { commandStreamReceiver.makeResident(*kernelIsaAllocation); @@ -1693,7 +1704,19 @@ cl_int Kernel::setArgImageWithMipLevel(uint32_t argIndex, imageFromBufferArgsCount += (pImage->isImageFromBuffer() ? 1 : 0) - (wasImageFromBuffer ? 1 : 0); void *surfaceState = nullptr; - if (isValidOffset(argAsImg.bindless)) { + if (isValidOffset(argAsImg.bindless) && pImage->isBindlessImage()) { + auto bindlessSlot = pImage->getBindlessSlot(); + if (bindlessSlot && bindlessSlot->ssPtr) { + surfaceState = bindlessSlot->ssPtr; + + auto &gfxCoreHelper = this->getGfxCoreHelper(); + auto patchLocation = ptrOffset(getCrossThreadData(), argAsImg.bindless); + uint64_t patchValue = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue( + static_cast(bindlessSlot->surfaceStateOffset)); + uint32_t patchSize = NEO::isUndefinedOffset(argAsImg.size) ? 0 : argAsImg.size; + patchWithRequiredSize(reinterpret_cast(patchLocation), patchSize, patchValue); + } + } else if (isValidOffset(argAsImg.bindless)) { auto ssIndex = getSurfaceStateIndexForBindlessOffset(argAsImg.bindless); if (ssIndex < std::numeric_limits::max()) { auto &gfxCoreHelper = this->getGfxCoreHelper(); @@ -1705,10 +1728,8 @@ cl_int Kernel::setArgImageWithMipLevel(uint32_t argIndex, surfaceState = ptrOffset(getSurfaceStateHeap(), argAsImg.bindful); } - // Sets SS structure UNRECOVERABLE_IF(surfaceState == nullptr); pImage->setImageArg(surfaceState, arg.getExtendedTypeInfo().isMediaBlockImage, mipLevel, rootDeviceIndex); - auto &imageDesc = pImage->getImageDesc(); auto &imageFormat = pImage->getImageFormat(); auto graphicsAllocation = pImage->getGraphicsAllocation(rootDeviceIndex); @@ -2048,13 +2069,22 @@ void Kernel::patchBindlessSurfaceStatesInCrossThreadData(uint64_t bindlessSurfac auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(); auto *crossThreadDataPtr = reinterpret_cast(getCrossThreadData()); - for (auto &arg : kernelInfo.kernelDescriptor.payloadMappings.explicitArgs) { + const auto &explicitArgs = kernelInfo.kernelDescriptor.payloadMappings.explicitArgs; + for (size_t argIndex = 0; argIndex < explicitArgs.size(); argIndex++) { + const auto &arg = explicitArgs[argIndex]; auto offset = NEO::undefined; if (arg.type == NEO::ArgDescriptor::argTPointer) { offset = arg.as().bindless; } else if (arg.type == NEO::ArgDescriptor::argTImage) { offset = arg.as().bindless; + if (NEO::isValidOffset(offset)) { + auto clMem = static_cast(kernelArguments[argIndex].object); + auto pImage = castToObject(clMem); + if (pImage && pImage->isBindlessImage()) { + continue; + } + } } else { continue; } diff --git a/opencl/source/mem_obj/image.cpp b/opencl/source/mem_obj/image.cpp index 5d2d8541f296f..914e8859cf918 100644 --- a/opencl/source/mem_obj/image.cpp +++ b/opencl/source/mem_obj/image.cpp @@ -300,6 +300,19 @@ Image *Image::create(Context *context, setImageProperties(image, *imageDesc, imgInfo, parentImage, parentBuffer, hostPtrRowPitch, hostPtrSlicePitch, imageCount, hostPtrMinSize); + auto defaultRootDeviceEnv = defaultDevice->getExecutionEnvironment()->rootDeviceEnvironments[defaultRootDeviceIndex].get(); + auto bindlessHelper = defaultRootDeviceEnv->getBindlessHeapsHelper(); + if (bindlessHelper && image && memoryProperties.flags.bindlessImage) { + auto allocation = image->getGraphicsAllocation(defaultRootDeviceIndex); + auto memManager = context->getMemoryManager(); + if (memManager->allocateBindlessSlot(allocation)) { + if (allocation->getBindlessOffset() != std::numeric_limits::max()) { + image->bindlessInfo = std::make_unique(allocation->getBindlessInfo()); + image->bindlessImage = true; + } + } + } + errcodeRet = CL_SUCCESS; auto &defaultHwInfo = defaultDevice->getHardwareInfo(); if (context->isProvidingPerformanceHints()) { @@ -842,7 +855,11 @@ cl_int Image::getImageInfo(cl_image_info paramName, srcParamSize = sizeof(cl_uint); srcParam = &(imageDesc.num_samples); break; - + case CL_IMAGE_BINDLESS_HANDLE_INTEL: + srcParamSize = sizeof(uint64_t); + retParam = getBindlessHandle(); + srcParam = &retParam; + break; default: getOsSpecificImageInfo(paramName, &srcParamSize, &srcParam); break; diff --git a/opencl/source/mem_obj/image.h b/opencl/source/mem_obj/image.h index 55b8f7d3b2b95..cc61e210a1174 100644 --- a/opencl/source/mem_obj/image.h +++ b/opencl/source/mem_obj/image.h @@ -6,6 +6,7 @@ */ #pragma once +#include "shared/source/helpers/bindless_heaps_helper.h" #include "shared/source/memory_manager/graphics_allocation.h" #include "opencl/source/helpers/surface_formats.h" @@ -205,6 +206,9 @@ class Image : public MemObj { void fillImageRegion(size_t *region) const; static bool validateHandleType(MemoryProperties &memoryProperties, UnifiedSharingMemoryDescription &extMem); + SurfaceStateInHeapInfo *getBindlessSlot() const { return bindlessInfo.get(); } + uint64_t getBindlessHandle() const { return bindlessInfo ? bindlessInfo->surfaceStateOffset : 0; } + bool isBindlessImage() const { return bindlessImage; } void setAs3DUavOrRtvImage(bool isUavOrRtv); void setIsPackedFormat(bool isPackedFormat) { this->isPackedFormat = isPackedFormat; } @@ -248,6 +252,8 @@ class Image : public MemObj { ImagePlane plane = ImagePlane::noPlane; bool is3DUAVOrRTV = false; bool isPackedFormat = false; + std::unique_ptr bindlessInfo; + bool bindlessImage = false; static bool isValidSingleChannelFormat(const cl_image_format *imageFormat); static bool isValidIntensityFormat(const cl_image_format *imageFormat); diff --git a/opencl/source/program/build.cpp b/opencl/source/program/build.cpp index 99e7ed863d0db..73b6c51a192a3 100644 --- a/opencl/source/program/build.cpp +++ b/opencl/source/program/build.cpp @@ -55,7 +55,14 @@ cl_int Program::build( options = ""; } } - + const bool hasBindlessImages = CompilerOptions::extract(CompilerOptions::bindlessImages, options); + const bool hasBindlessAdvancedMode = CompilerOptions::extract(CompilerOptions::bindlessAdvancedMode, options); + if (hasBindlessImages) { + CompilerOptions::concatenateAppend(internalOptions, CompilerOptions::bindlessImages); + } + if (hasBindlessAdvancedMode) { + CompilerOptions::concatenateAppend(internalOptions, CompilerOptions::bindlessAdvancedMode); + } const bool shouldSuppressRebuildWarning{CompilerOptions::extract(CompilerOptions::noRecompiledFromIr, options)}; extractInternalOptions(options, internalOptions); CompilerOptions::applyAdditionalApiOptions(options); diff --git a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp index 10547f82f8d33..9ea5757598b5a 100644 --- a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp +++ b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp @@ -8,7 +8,11 @@ #include "shared/source/helpers/api_specific_config.h" #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/compression_selector.h" +#include "shared/source/release_helper/release_helper.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/mocks/mock_device.h" +#include "shared/test/common/mocks/mock_release_helper.h" +#include "shared/test/common/mocks/ult_device_factory.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -57,4 +61,24 @@ TEST(ApiSpecificConfigOclTests, WhenCheckingIfDeviceUsmPoolingIsEnabledThenRetur EXPECT_TRUE(ApiSpecificConfig::isDeviceUsmPoolingEnabled()); } +TEST(ApiSpecificConfigOclTests, WhenGettingGlobalBindlessHeapConfigurationWithDebugFlagThenReturnDebugFlagValue) { + DebugManagerStateRestore restorer; + MockReleaseHelper releaseHelper; + + debugManager.flags.UseExternalAllocatorForSshAndDsh.set(1); + EXPECT_TRUE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(releaseHelper)); + + debugManager.flags.UseExternalAllocatorForSshAndDsh.set(0); + EXPECT_FALSE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(releaseHelper)); +} + +TEST(ApiSpecificConfigOclTests, WhenGettingBindlessModeThenQueryHardwareCapability) { + DebugManagerStateRestore restorer; + UltDeviceFactory deviceFactory{1, 0}; + auto *device = deviceFactory.rootDevices[0]; + + debugManager.flags.UseBindlessMode.set(-1); + auto result = ApiSpecificConfig::getBindlessMode(static_cast(*device)); + EXPECT_EQ(result, device->getReleaseHelper().isGlobalBindlessAllocatorEnabled()); +} } // namespace NEO diff --git a/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp b/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp index ca4b1d3c2ee03..7c950592ae2db 100644 --- a/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp +++ b/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp @@ -14,6 +14,7 @@ #include "shared/test/common/mocks/mock_graphics_allocation.h" #include "shared/test/common/mocks/ult_device_factory.h" +#include "opencl/extensions/public/cl_ext_private.h" #include "opencl/source/helpers/cl_memory_properties_helpers.h" #include "opencl/source/mem_obj/mem_obj_helper.h" #include "opencl/test/unit_test/mocks/mock_cl_device.h" @@ -599,3 +600,20 @@ TEST_F(MemoryPropertiesHelperTests, givenSubDeviceIdWhenParsingExtraMemoryProper EXPECT_EQ(0b10u, memoryProperties.pDevice->getDeviceBitfield().to_ulong()); EXPECT_EQ(&context.pSubDevice1->getDevice(), memoryProperties.pDevice); } +TEST_F(MemoryPropertiesHelperTests, givenBindlessImagePropertyWhenParsingMemoryPropertiesForImageThenTrueIsReturnedAndFlagIsSet) { + cl_mem_properties_intel properties[] = { + CL_MEM_BINDLESS_IMAGE_INTEL, 1, + 0}; + EXPECT_TRUE(ClMemoryPropertiesHelper::parseMemoryProperties(properties, memoryProperties, flags, flagsIntel, allocflags, + ClMemoryPropertiesHelper::ObjType::image, context)); + EXPECT_TRUE(memoryProperties.flags.bindlessImage); +} + +TEST_F(MemoryPropertiesHelperTests, givenNoBindlessImagePropertyWhenParsingMemoryPropertiesForImageThenBindlessFlagIsNotSet) { + cl_mem_properties_intel properties[] = { + CL_MEM_FLAGS, CL_MEM_READ_WRITE, + 0}; + EXPECT_TRUE(ClMemoryPropertiesHelper::parseMemoryProperties(properties, memoryProperties, flags, flagsIntel, allocflags, + ClMemoryPropertiesHelper::ObjType::image, context)); + EXPECT_FALSE(memoryProperties.flags.bindlessImage); +} diff --git a/shared/source/compiler_interface/compiler_options.h b/shared/source/compiler_interface/compiler_options.h index 3fe042adcf7cb..b4c6f256a0b17 100644 --- a/shared/source/compiler_interface/compiler_options.h +++ b/shared/source/compiler_interface/compiler_options.h @@ -32,6 +32,8 @@ inline constexpr ConstStringRef createLibrary = "-create-library"; inline constexpr ConstStringRef generateDebugInfo = "-g"; inline constexpr ConstStringRef generateSourcePath = "-s"; inline constexpr ConstStringRef bindlessMode = "-cl-intel-use-bindless-mode -cl-intel-use-bindless-advanced-mode"; +inline constexpr ConstStringRef bindlessImages = "-cl-intel-use-bindless-images"; +inline constexpr ConstStringRef bindlessAdvancedMode = "-cl-intel-use-bindless-advanced-mode"; inline constexpr ConstStringRef uniformWorkgroupSize = "-cl-uniform-work-group-size"; inline constexpr ConstStringRef forceEmuInt32DivRem = "-cl-intel-force-emu-int32divrem"; inline constexpr ConstStringRef forceEmuInt32DivRemSP = "-cl-intel-force-emu-sp-int32divrem"; diff --git a/shared/source/device/root_device.cpp b/shared/source/device/root_device.cpp index 4174d07f0c02e..e4325f61a2695 100644 --- a/shared/source/device/root_device.cpp +++ b/shared/source/device/root_device.cpp @@ -54,8 +54,7 @@ void RootDevice::createBindlessHeapsHelper() { EnvironmentVariableReader envReader; bool disableGlobalBindless = envReader.getSetting("NEO_L0_SYSMAN_NO_CONTEXT_MODE", false); - - if (!disableGlobalBindless && ApiSpecificConfig::getGlobalBindlessHeapConfiguration(this->getReleaseHelper()) && ApiSpecificConfig::getBindlessMode(*this)) { + if (!disableGlobalBindless && ApiSpecificConfig::getGlobalBindlessHeapConfiguration(this->getReleaseHelper())) { this->executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]->createBindlessHeapsHelper(this, getNumGenericSubDevices() > 1); } } diff --git a/shared/source/helpers/memory_properties_flags.h b/shared/source/helpers/memory_properties_flags.h index 5611a3c206d64..4d2c03667501d 100644 --- a/shared/source/helpers/memory_properties_flags.h +++ b/shared/source/helpers/memory_properties_flags.h @@ -35,6 +35,7 @@ struct MemoryFlags { uint32_t compressedHint : 1; uint32_t uncompressedHint : 1; uint32_t ipcSupportedAllocationByDefault : 1; + uint32_t bindlessImage : 1; bool operator==(const MemoryFlags &) const = default; };