/*
 * Copyright (C) 2020-2026 Intel Corporation
 *
 * SPDX-License-Identifier: MIT
 *
 */

#include "shared/source/command_container/command_encoder.h"
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/command_stream/submissions_aggregator.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/direct_submission/direct_submission_controller.h"
#include "shared/source/direct_submission/direct_submission_hw.h"
#include "shared/source/direct_submission/relaxed_ordering_helper.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/helpers/aligned_memory.h"
#include "shared/source/helpers/async_launch_mode.h"
#include "shared/source/helpers/compiler_product_helper.h"
#include "shared/source/helpers/flush_stamp.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/memory_manager/allocation_properties.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/memory_manager/memory_operations_handler.h"
#include "shared/source/os_interface/os_context.h"
#include "shared/source/os_interface/product_helper.h"

#include "create_direct_submission_hw.inl"

#include <algorithm>
#include <cstring>
#include <future>

namespace NEO {

template <typename GfxFamily, typename Dispatcher>
DirectSubmissionHw<GfxFamily, Dispatcher>::DirectSubmissionHw(const DirectSubmissionInputParams &inputParams)
    : ringBuffers(RingBufferUse::initialRingBufferCount), csr(inputParams.csr), osContext(inputParams.osContext), rootDeviceIndex(inputParams.rootDeviceIndex), rootDeviceEnvironment(inputParams.rootDeviceEnvironment) {
    memoryManager = inputParams.memoryManager;
    globalFenceAllocation = inputParams.globalFenceAllocation;
    hwInfo = inputParams.rootDeviceEnvironment.getHardwareInfo();
    memoryOperationHandler = inputParams.rootDeviceEnvironment.memoryOperationsInterface.get();

    auto &productHelper = inputParams.rootDeviceEnvironment.getHelper<ProductHelper>();
    auto *releaseHelper = inputParams.rootDeviceEnvironment.getReleaseHelper();
    auto &compilerProductHelper = inputParams.rootDeviceEnvironment.getHelper<CompilerProductHelper>();

    if (debugManager.flags.DirectSubmissionMaxRingBuffers.get() != -1) {
        this->maxRingBufferCount = debugManager.flags.DirectSubmissionMaxRingBuffers.get();
    }

    if (debugManager.flags.DirectSubmissionDetectGpuHang.get() != -1) {
        detectGpuHang = !!debugManager.flags.DirectSubmissionDetectGpuHang.get();
    }

    miMemFenceRequired = productHelper.isAcquireGlobalFenceInDirectSubmissionRequired(*hwInfo);

    if (debugManager.flags.DirectSubmissionInsertExtraMiMemFenceCommands.get() != -1) {
        miMemFenceRequired = debugManager.flags.DirectSubmissionInsertExtraMiMemFenceCommands.get();
    }

    if (miMemFenceRequired && compilerProductHelper.isHeaplessModeEnabled(*hwInfo)) {
        this->systemMemoryFenceAddressSet = true;
    }

    isDisablePrefetcherRequired = productHelper.isPrefetcherDisablingInDirectSubmissionRequired();
    if (debugManager.flags.DirectSubmissionDisablePrefetcher.get() != -1) {
        isDisablePrefetcherRequired = !!debugManager.flags.DirectSubmissionDisablePrefetcher.get();
    }

    setImmWritePostSyncOffset();

    dcFlushRequired = MemorySynchronizationCommands<GfxFamily>::getDcFlushEnable(true, inputParams.rootDeviceEnvironment);
    auto &gfxCoreHelper = inputParams.rootDeviceEnvironment.getHelper<GfxCoreHelper>();
    relaxedOrderingEnabled = gfxCoreHelper.isRelaxedOrderingSupported();

    this->currentRelaxedOrderingQueueSize = RelaxedOrderingHelper::queueSizeMultiplier;

    if (debugManager.flags.DirectSubmissionRelaxedOrdering.get() != -1) {
        relaxedOrderingEnabled = (debugManager.flags.DirectSubmissionRelaxedOrdering.get() == 1);
    }

    if (Dispatcher::isCopy() && relaxedOrderingEnabled) {
        relaxedOrderingEnabled = (debugManager.flags.DirectSubmissionRelaxedOrderingForBcs.get() != 0);
    }

    this->isSwitchOnUnsuccessful = false;
    if (!this->osContext.isExclusivelyHpContext()) {
        if (this->osContext.isHighPriority()) {
            this->isSwitchOnUnsuccessful = true;
        } else if (this->osContext.hasPriorityLevel()) {
            this->isSwitchOnUnsuccessful = this->osContext.getPriorityLevel() == gfxCoreHelper.getHwQueuePriority(gfxCoreHelper.getHighestQueuePriorityLevel());
        }
    }
    if (debugManager.flags.DirectSubmissionSwitchSemaphoreMode.get() != -1) {
        this->isSwitchOnUnsuccessful = !!debugManager.flags.DirectSubmissionSwitchSemaphoreMode.get();
    }

    currentQueueWorkCount = getInitialSemaphoreValue();
    this->useSemaphore64bCmd = productHelper.isAvailableSemaphore64(releaseHelper, *inputParams.rootDeviceEnvironment.getHardwareInfo());
}

template <typename GfxFamily, typename Dispatcher>
DirectSubmissionHw<GfxFamily, Dispatcher>::~DirectSubmissionHw() = default;

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::allocateResources() {
    DirectSubmissionAllocations allocations;

    for (uint32_t ringBufferIndex = 0; ringBufferIndex < RingBufferUse::initialRingBufferCount; ringBufferIndex++) {
        auto ringBuffer = allocateRingBuffer();
        this->ringBuffers[ringBufferIndex].ringBuffer = ringBuffer;
        UNRECOVERABLE_IF(ringBuffer == nullptr);
        allocations.push_back(ringBuffer);
        memset(ringBuffer->getUnderlyingBuffer(), 0, ringBuffer->getUnderlyingBufferSize());
    }
    bool isMultiOsContextCapable = osContext.getNumSupportedDevices() > 1u;
    const AllocationProperties semaphoreAllocationProperties{rootDeviceIndex,
                                                             true, MemoryConstants::pageSize,
                                                             AllocationType::semaphoreBuffer,
                                                             isMultiOsContextCapable, false, osContext.getDeviceBitfield()};
    semaphores = memoryManager->allocateGraphicsMemoryWithProperties(semaphoreAllocationProperties);
    UNRECOVERABLE_IF(semaphores == nullptr);
    allocations.push_back(semaphores);

    if (this->workPartitionAllocation != nullptr) {
        allocations.push_back(workPartitionAllocation);
    }

    if (completionFenceAllocation != nullptr) {
        allocations.push_back(completionFenceAllocation);
    }

    if (this->relaxedOrderingEnabled) {
        const AllocationProperties allocationProperties(rootDeviceIndex,
                                                        true, MemoryConstants::pageSize64k,
                                                        AllocationType::deferredTasksList,
                                                        isMultiOsContextCapable, false, osContext.getDeviceBitfield());

        deferredTasksListAllocation = memoryManager->allocateGraphicsMemoryWithProperties(allocationProperties);
        UNRECOVERABLE_IF(deferredTasksListAllocation == nullptr);

        allocations.push_back(deferredTasksListAllocation);

        AllocationProperties relaxedOrderingSchedulerAllocationProperties(rootDeviceIndex,
                                                                          true, MemoryConstants::pageSize64k,
                                                                          AllocationType::commandBuffer,
                                                                          isMultiOsContextCapable, false, osContext.getDeviceBitfield());
        relaxedOrderingSchedulerAllocationProperties.flags.cantBeReadOnly = true;
        relaxedOrderingSchedulerAllocation = memoryManager->allocateGraphicsMemoryWithProperties(relaxedOrderingSchedulerAllocationProperties);
        UNRECOVERABLE_IF(relaxedOrderingSchedulerAllocation == nullptr);

        allocations.push_back(relaxedOrderingSchedulerAllocation);
    }

    if (debugManager.flags.DirectSubmissionPrintBuffers.get()) {
        for (uint32_t ringBufferIndex = 0; ringBufferIndex < RingBufferUse::initialRingBufferCount; ringBufferIndex++) {
            const auto ringBuffer = this->ringBuffers[ringBufferIndex].ringBuffer;

            PRINT_STRING(true, stdout, "Ring buffer %u - gpu address: %" PRIx64 " - %" PRIx64 ", cpu address: %p - %p, size: %zu \n",
                         ringBufferIndex,
                         ringBuffer->getGpuAddress(),
                         ptrOffset(ringBuffer->getGpuAddress(), ringBuffer->getUnderlyingBufferSize()),
                         ringBuffer->getUnderlyingBuffer(),
                         ptrOffset(ringBuffer->getUnderlyingBuffer(), ringBuffer->getUnderlyingBufferSize()),
                         ringBuffer->getUnderlyingBufferSize());
        }
    }

    handleResidency(nullptr);
    ringCommandStream.replaceBuffer(this->ringBuffers[0u].ringBuffer->getUnderlyingBuffer(), minimumRingRequiredSize);
    ringCommandStream.replaceGraphicsAllocation(this->ringBuffers[0].ringBuffer);

    semaphorePtr = semaphores->getUnderlyingBuffer();
    semaphoreGpuVa = semaphores->getGpuAddress();
    semaphoreData = static_cast<volatile RingSemaphoreData *>(semaphorePtr);
    memset(semaphorePtr, 0, sizeof(RingSemaphoreData));
    semaphoreData->queueWorkCount = 0;

    this->gpuVaForMiFlush = this->semaphoreGpuVa + offsetof(RingSemaphoreData, miFlushSpace);
    this->gpuVaForPagingFenceSemaphore = this->semaphoreGpuVa + offsetof(RingSemaphoreData, pagingFenceCounter);
    auto ret = makeResourcesResident(allocations);

    return ret && allocateOsResources();
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::makeResourcesResident(DirectSubmissionAllocations &allocations) {
    auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef<GraphicsAllocation *>(allocations), false, false, true) == MemoryOperationsStatus::success;

    return ret;
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::allocateOsResources() {
    if (this->semaphorePtr != nullptr) {
        this->tagAddress = reinterpret_cast<volatile TagAddressType *>(reinterpret_cast<uint8_t *>(this->semaphorePtr) + offsetof(RingSemaphoreData, tagAllocation));
    } else {
        this->tagAddress = nullptr;
    }
    return true;
}

template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::unblockGpu() {
    SemaphoreFenceHelper fence(*this);

    if (this->pciBarrierPtr) {
        *this->pciBarrierPtr = 0u;
    }

    PRINT_STRING(debugManager.flags.DirectSubmissionPrintSemaphoreUsage.get() == 1, stdout, "DirectSubmission semaphore %" PRIx64 " unlocked with value: %u\n", semaphoreGpuVa, currentQueueWorkCount);

    semaphoreData->queueWorkCount = currentQueueWorkCount;
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::initialize(bool submitOnInit) {
    bool ret = allocateResources();

    if (ret && submitOnInit) {
        size_t startBufferSize = Dispatcher::getSizePreemption() +
                                 getSizeSemaphoreSection(false);

        Dispatcher::dispatchPreemption(ringCommandStream);
        if (this->partitionedMode) {
            startBufferSize += getSizePartitionRegisterConfigurationSection();
            dispatchPartitionRegisterConfiguration();

            this->partitionConfigSet = true;
        }
        if (this->globalFenceAllocation && !this->systemMemoryFenceAddressSet) {
            startBufferSize += getSizeSystemMemoryFenceAddress();
            dispatchSystemMemoryFenceAddress();

            this->systemMemoryFenceAddressSet = true;
        }
        if (this->relaxedOrderingEnabled) {
            preinitializeRelaxedOrderingSections();

            initRelaxedOrderingRegisters();
            dispatchStaticRelaxedOrderingScheduler();
            startBufferSize += RelaxedOrderingHelper::getSizeRegistersInit<GfxFamily>();

            this->relaxedOrderingInitialized = true;
        }

        dispatchSemaphoreSection(currentQueueWorkCount);

        ringStart = submit(ringCommandStream.getGraphicsAllocation()->getGpuAddress(), startBufferSize, nullptr);
        return ringStart;
    }
    return ret;
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::stopRingBuffer(bool blocking) {
    if (!ringStart) {
        if (blocking) {
            this->ensureRingCompletion();
        }
        return true;
    }

    if (this->relaxedOrderingEnabled && this->relaxedOrderingSchedulerRequired) {
        dispatchRelaxedOrderingQueueStall();
    }

    dispatchStopRingBufferSection();
    Dispatcher::dispatchStopCommandBuffer(ringCommandStream);

    auto bytesToPad = Dispatcher::getSizeStartCommandBuffer() - Dispatcher::getSizeStopCommandBuffer();
    EncodeNoop<GfxFamily>::emitNoop(ringCommandStream, bytesToPad);
    EncodeNoop<GfxFamily>::alignToCacheLine(ringCommandStream);

    this->unblockGpu();

    this->handleStopRingBuffer();
    this->ringStart = false;

    if (blocking) {
        this->ensureRingCompletion();
    }

    return true;
}

template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchSemaphoreSection(uint32_t value) {
    using COMPARE_OPERATION = typename GfxFamily::MI_SEMAPHORE_WAIT::COMPARE_OPERATION;

    PRINT_STRING(debugManager.flags.DirectSubmissionPrintSemaphoreUsage.get() == 1, stdout,
                 "DirectSubmission semaphore %" PRIx64 " programmed with value: %u\n", semaphoreGpuVa, value);

    dispatchDisablePrefetcher(true);

    if (this->relaxedOrderingEnabled && this->relaxedOrderingSchedulerRequired) {
        dispatchRelaxedOrderingSchedulerSection(value);
    } else {
        EncodeSemaphore<GfxFamily>::addMiSemaphoreWaitCommand(ringCommandStream,
                                                              semaphoreGpuVa,
                                                              value,
                                                              COMPARE_OPERATION::COMPARE_OPERATION_SAD_GREATER_THAN_OR_EQUAL_SDD, false, false, false, this->isSwitchOnUnsuccessful, this->useSemaphore64bCmd, nullptr);
    }

    if (miMemFenceRequired) {
        MemorySynchronizationCommands<GfxFamily>::addAdditionalSynchronizationForDirectSubmission(ringCommandStream, this->gpuVaForAdditionalSynchronizationWA, NEO::FenceType::acquire, rootDeviceEnvironment);
    }

    dispatchPrefetchMitigation();
    dispatchDisablePrefetcher(false);
}

template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeSemaphoreSection(bool relaxedOrderingSchedulerRequired) {
    size_t semaphoreSize = (this->relaxedOrderingEnabled && relaxedOrderingSchedulerRequired) ? RelaxedOrderingHelper::DynamicSchedulerSizeAndOffsetSection<GfxFamily>::getTotalSize()
                                                                                              : EncodeSemaphore<GfxFamily>::getSizeMiSemaphoreWait();
    semaphoreSize += getSizePrefetchMitigation();

    if (isDisablePrefetcherRequired) {
        semaphoreSize += 2 * getSizeDisablePrefetcher();
    }

    if (miMemFenceRequired) {
        semaphoreSize += MemorySynchronizationCommands<GfxFamily>::getSizeForSingleAdditionalSynchronizationForDirectSubmission(rootDeviceEnvironment);
    }

    return semaphoreSize;
}

template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchStartSection(uint64_t gpuStartAddress) {
    Dispatcher::dispatchStartCommandBuffer(ringCommandStream, gpuStartAddress);
}

template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeStartSection() {
    return Dispatcher::getSizeStartCommandBuffer();
}

template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchSwitchRingBufferSection(uint64_t nextBufferGpuAddress) {
    TagData currentTagData = {};
    getTagAddressValueForRingSwitch(currentTagData);
    Dispatcher::dispatchMonitorFence(ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, this->notifyKmdDuringMonitorFence);
    Dispatcher::dispatchStartCommandBuffer(ringCommandStream, nextBufferGpuAddress);
}

template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeSwitchRingBufferSection() {
    return Dispatcher::getSizeStartCommandBuffer() + Dispatcher::getSizeMonitorFence(rootDeviceEnvironment);
}

template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeEnd(bool relaxedOrderingSchedulerRequired) {
    size_t size = Dispatcher::getSizeStopCommandBuffer() +
                  (Dispatcher::getSizeStartCommandBuffer() - Dispatcher::getSizeStopCommandBuffer()) +
                  MemoryConstants::cacheLineSize + dispatchStopRingBufferSectionSize();
    if (this->relaxedOrderingEnabled && relaxedOrderingSchedulerRequired) {
        size += getSizeDispatchRelaxedOrderingQueueStall();
    }
    return size;
}

template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeDispatch(bool relaxedOrderingSchedulerRequired, bool returnPtrsRequired, bool dispatchMonitorFence) {
    size_t size = getSizeSemaphoreSection(relaxedOrderingSchedulerRequired) + getSizeStartSection();
    if (this->relaxedOrderingEnabled && returnPtrsRequired) {
        size += RelaxedOrderingHelper::getSizeReturnPtrRegs<GfxFamily>();
    }

    if (dispatchMonitorFence) {
        size += Dispatcher::getSizeMonitorFence(rootDeviceEnvironment);
    }

    size += getSizeNewResourceHandler();

    return size;
}

template <typename GfxFamily, typename Dispatcher>
void *DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchWorkloadSection(BatchBuffer &batchBuffer, bool dispatchMonitorFence) {
    void *currentPosition = ringCommandStream.getSpace(0);
    auto copyCmdBuffer = this->copyCommandBufferIntoRing(batchBuffer);

    if (debugManager.flags.DirectSubmissionPrintBuffers.get()) {
        std::ostringstream ss;
        // clang-format off
        ss << std::showbase
        << "Client buffer:\n"
        << "Command buffer allocation - gpu address: " << std::hex
            << batchBuffer.commandBufferAllocation->getGpuAddress()
            << " - "
            << ptrOffset(batchBuffer.commandBufferAllocation->getGpuAddress(), batchBuffer.commandBufferAllocation->getUnderlyingBufferSize())
            << ", cpu address: "
            << batchBuffer.commandBufferAllocation->getUnderlyingBuffer()
            << " - "
            << ptrOffset(batchBuffer.commandBufferAllocation->getUnderlyingBuffer(), batchBuffer.commandBufferAllocation->getUnderlyingBufferSize())
            << ", size: " << std::dec
            << batchBuffer.commandBufferAllocation->getUnderlyingBufferSize()
            << "\n"
        << "Command buffer - start gpu address: " << std::hex
            << ptrOffset(batchBuffer.commandBufferAllocation->getGpuAddress(), batchBuffer.startOffset)
            << " - "
            << ptrOffset(batchBuffer.commandBufferAllocation->getGpuAddress(), batchBuffer.usedSize)
            << ", start cpu address: "
            << ptrOffset(batchBuffer.commandBufferAllocation->getUnderlyingBuffer(), batchBuffer.startOffset)
            << " - "
            << ptrOffset(batchBuffer.commandBufferAllocation->getUnderlyingBuffer(), batchBuffer.usedSize)
            << ", start offset: " << std::dec
            << batchBuffer.startOffset
            << ", used size: "
            << batchBuffer.usedSize
            << "\n"
        << "Ring buffer for submission - start gpu address: " << std::hex
            << ringCommandStream.getGraphicsAllocation()->getGpuAddress()
            << " - "
            << ptrOffset(ringCommandStream.getGraphicsAllocation()->getGpuAddress(), ringCommandStream.getGraphicsAllocation()->getUnderlyingBufferSize())
            << ", start cpu address: "
            << ringCommandStream.getGraphicsAllocation()->getUnderlyingBuffer()
            << " - "
            << ptrOffset(ringCommandStream.getGraphicsAllocation()->getUnderlyingBuffer(), ringCommandStream.getGraphicsAllocation()->getUnderlyingBufferSize())
            << ", size: " << std::dec
            << ringCommandStream.getGraphicsAllocation()->getUnderlyingBufferSize()
            << ",  submission address: " << std::hex
            << ptrOffset(ringCommandStream.getGraphicsAllocation()->getGpuAddress(), ringCommandStream.getUsed())
            << ", used size: " << std::dec
            << ringCommandStream.getUsed()
            << ", copyCmdBuffer: "
            << copyCmdBuffer
            << "\n";
        // clang-format on
        PRINT_STRING(true, stdout, "%s", ss.str().c_str());
    }

    if (batchBuffer.pagingFenceSemInfo.requiresProgrammingSemaphore()) {
        dispatchSemaphoreForPagingFence(batchBuffer.pagingFenceSemInfo.pagingFenceValue);
    }

    auto commandStreamAddress = ptrOffset(batchBuffer.commandBufferAllocation->getGpuAddress(), batchBuffer.startOffset);
    void *returnCmd = batchBuffer.endCmdPtr;

    LinearStream relaxedOrderingReturnPtrCmdStream;
    if (this->relaxedOrderingEnabled && batchBuffer.hasRelaxedOrderingDependencies) {
        // preallocate and patch after start section
        auto relaxedOrderingReturnPtrCmds = ringCommandStream.getSpace(RelaxedOrderingHelper::getSizeReturnPtrRegs<GfxFamily>());
        relaxedOrderingReturnPtrCmdStream.replaceBuffer(relaxedOrderingReturnPtrCmds, RelaxedOrderingHelper::getSizeReturnPtrRegs<GfxFamily>());
    }

    if (copyCmdBuffer) {
        auto cmdStreamTaskPtr = ptrOffset(batchBuffer.stream->getCpuBase(), batchBuffer.startOffset);
        auto sizeToCopy = ptrDiff(returnCmd, cmdStreamTaskPtr);
        auto ringPtr = ringCommandStream.getSpace(sizeToCopy);
        memcpy(ringPtr, cmdStreamTaskPtr, sizeToCopy);
        batchBuffer.stream->reclaimSpace(batchBuffer.usedSize - batchBuffer.startOffset);
    } else {
        dispatchStartSection(commandStreamAddress);
    }

    uint64_t returnGpuPointer = ringCommandStream.getCurrentGpuAddressPosition();

    if (this->relaxedOrderingEnabled && batchBuffer.hasRelaxedOrderingDependencies) {
        dispatchRelaxedOrderingReturnPtrRegs(relaxedOrderingReturnPtrCmdStream, returnGpuPointer);
    } else if (!copyCmdBuffer) {
        setReturnAddress(returnCmd, returnGpuPointer);
    }

    if (this->relaxedOrderingEnabled && batchBuffer.hasRelaxedOrderingDependencies) {
        dispatchTaskStoreSection(batchBuffer.taskStartAddress);

        uint32_t expectedQueueSize = batchBuffer.numCsrClients * RelaxedOrderingHelper::queueSizeMultiplier;
        expectedQueueSize = std::min(expectedQueueSize, RelaxedOrderingHelper::maxQueueSize);

        if (expectedQueueSize > this->currentRelaxedOrderingQueueSize && debugManager.flags.DirectSubmissionRelaxedOrderingQueueSizeLimit.get() == -1) {
            updateRelaxedOrderingQueueSize(expectedQueueSize);
        }
    }

    if (dispatchMonitorFence) {
        TagData currentTagData = {};
        getTagAddressValue(currentTagData);
        Dispatcher::dispatchMonitorFence(ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, this->notifyKmdDuringMonitorFence);
    }

    dispatchSemaphoreSection(currentQueueWorkCount + 1);
    return currentPosition;
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::copyCommandBufferIntoRing(BatchBuffer &batchBuffer) {
    /* Command buffer can't be copied into ring if implicit scaling or metrics are enabled,
       because those features uses GPU VAs of command buffer which would be invalid after copy. */

    auto ret = !batchBuffer.disableFlatRingBuffer &&
               this->osContext.getNumSupportedDevices() == 1u &&
               !this->rootDeviceEnvironment.executionEnvironment.areMetricsEnabled() &&
               !batchBuffer.chainedBatchBuffer &&
               batchBuffer.commandBufferAllocation &&
               MemoryPoolHelper::isSystemMemoryPool(batchBuffer.commandBufferAllocation->getMemoryPool()) &&
               !batchBuffer.hasRelaxedOrderingDependencies;

    if (debugManager.flags.DirectSubmissionFlatRingBuffer.get() != -1) {
        ret &= !!debugManager.flags.DirectSubmissionFlatRingBuffer.get();
    }

    return ret;
}

template <typename GfxFamily, typename Dispatcher>
size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getUllsStateSize() {
    size_t startSize = 0u;
    if (!this->partitionConfigSet) {
        startSize += getSizePartitionRegisterConfigurationSection();
    }
    if (this->miMemFenceRequired && !this->systemMemoryFenceAddressSet) {
        startSize += getSizeSystemMemoryFenceAddress();
    }
    if (this->relaxedOrderingEnabled && !this->relaxedOrderingInitialized) {
        startSize += RelaxedOrderingHelper::getSizeRegistersInit<GfxFamily>();
    }
    return startSize;
}

template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchUllsState() {
    if (!this->partitionConfigSet) {
        dispatchPartitionRegisterConfiguration();
        this->partitionConfigSet = true;
    }
    if (this->globalFenceAllocation && !this->systemMemoryFenceAddressSet) {
        dispatchSystemMemoryFenceAddress();
        this->systemMemoryFenceAddressSet = true;
    }
    if (this->relaxedOrderingEnabled && !this->relaxedOrderingInitialized) {
        preinitializeRelaxedOrderingSections();
        dispatchStaticRelaxedOrderingScheduler();
        initRelaxedOrderingRegisters();

        this->relaxedOrderingInitialized = true;
    }
}

template <typename GfxFamily, typename Dispatcher>
uint32_t DirectSubmissionHw<GfxFamily, Dispatcher>::getInitialSemaphoreValue() const {
    return debugManager.flags.DirectSubmissionInitialSemaphoreValue.getIfNotDefault<uint32_t>(1);
}

template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::handleSemaphoreDataOverflow() {
    stopRingBuffer(true);
    currentQueueWorkCount = 0;
    unblockGpu(); // set gpu allocation to 0

    currentQueueWorkCount = getInitialSemaphoreValue();
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchCommandBuffer(BatchBuffer &batchBuffer, FlushStampTracker &flushStamp) {
    // Handle overflow earlier (uint32_max - 3), in case of additional ring starts/stops
    if ((currentQueueWorkCount + 1) >= (std::numeric_limits<uint32_t>::max() - 3)) {
        handleSemaphoreDataOverflow();
    }

    this->handleRingRestartForUllsLightResidency(batchBuffer.allocationsForResidency);

    lastSubmittedThrottle = batchBuffer.throttle;
    bool relaxedOrderingSchedulerWillBeNeeded = (this->relaxedOrderingSchedulerRequired || batchBuffer.hasRelaxedOrderingDependencies);
    bool dispatchMonitorFence = this->dispatchMonitorFenceRequired(batchBuffer.dispatchMonitorFence);

    size_t dispatchSize = this->getUllsStateSize() + getSizeDispatch(relaxedOrderingSchedulerWillBeNeeded, batchBuffer.hasRelaxedOrderingDependencies, dispatchMonitorFence);

    if (this->copyCommandBufferIntoRing(batchBuffer)) {
        dispatchSize += (batchBuffer.stream->getUsed() - batchBuffer.startOffset) - 2 * getSizeStartSection();
    }
    if (batchBuffer.pagingFenceSemInfo.requiresProgrammingSemaphore()) {
        dispatchSize += getSizeSemaphoreForPagingFence();
    }

    size_t cycleSize = getSizeSwitchRingBufferSection();
    size_t requiredMinimalSize = dispatchSize + cycleSize + getSizeEnd(relaxedOrderingSchedulerWillBeNeeded);
    if (this->relaxedOrderingEnabled) {
        requiredMinimalSize += +RelaxedOrderingHelper::getSizeReturnPtrRegs<GfxFamily>();

        if (batchBuffer.hasStallingCmds && this->relaxedOrderingSchedulerRequired) {
            requiredMinimalSize += getSizeDispatchRelaxedOrderingQueueStall();
        }
        if (batchBuffer.hasRelaxedOrderingDependencies) {
            requiredMinimalSize += RelaxedOrderingHelper::getSizeTaskStoreSection<GfxFamily>() + sizeof(typename GfxFamily::MI_STORE_DATA_IMM);
        }
    }

    auto needStart = !this->ringStart;

    this->switchRingBuffersNeeded(requiredMinimalSize, batchBuffer.allocationsForResidency);

    auto startVA = ringCommandStream.getCurrentGpuAddressPosition();

    this->dispatchUllsState();

    if (this->relaxedOrderingEnabled && batchBuffer.hasStallingCmds && this->relaxedOrderingSchedulerRequired) {
        dispatchRelaxedOrderingQueueStall();
    }

    this->relaxedOrderingSchedulerRequired |= batchBuffer.hasRelaxedOrderingDependencies;

    handleNewResourcesSubmission();

    dispatchWorkloadSection(batchBuffer, dispatchMonitorFence);

    auto requiresBlockingResidencyHandling = batchBuffer.pagingFenceSemInfo.requiresBlockingResidencyHandling;
    if (!this->submitCommandBufferToGpu(needStart, startVA, requiredMinimalSize, requiresBlockingResidencyHandling, batchBuffer.allocationsForResidency)) {
        return false;
    }

    currentQueueWorkCount++;

    uint64_t flushValue = updateTagValue(dispatchMonitorFence);
    if (flushValue == DirectSubmissionHw<GfxFamily, Dispatcher>::updateTagValueFail) {
        return false;
    }
    flushStamp.setStamp(flushValue);

    return this->ringStart;
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::submitCommandBufferToGpu(bool needStart, uint64_t gpuAddress, size_t size, bool needWait, const ResidencyContainer *allocationsForResidency) {
    if (needStart) {
        this->ringStart = this->submit(gpuAddress, size, allocationsForResidency);
        if (auto controller = rootDeviceEnvironment.executionEnvironment.directSubmissionController.get()) {
            controller->notifyNewSubmission(&csr);
        }
        return this->ringStart;
    } else {
        if (needWait) {
            handleResidency(nullptr);
        }
        if (!this->csr.isHardwareMode()) {
            handleResidency(allocationsForResidency);
        }

        this->unblockGpu();
        return true;
    }
}

template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::setReturnAddress(void *returnCmd, uint64_t returnAddress) {
    using MI_BATCH_BUFFER_START = typename GfxFamily::MI_BATCH_BUFFER_START;

    MI_BATCH_BUFFER_START cmd = GfxFamily::cmdInitBatchBufferStart;
    cmd.setBatchBufferStartAddress(returnAddress);
    cmd.setAddressSpaceIndicator(MI_BATCH_BUFFER_START::ADDRESS_SPACE_INDICATOR_PPGTT);

    MI_BATCH_BUFFER_START *returnBBStart = static_cast<MI_BATCH_BUFFER_START *>(returnCmd);
    *returnBBStart = cmd;
}

template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::handleNewResourcesSubmission() {
    if (isNewResourceHandleNeeded()) {
        auto tlbFlushCounter = this->osContext.peekTlbFlushCounter();
        Dispatcher::dispatchTlbFlush(this->ringCommandStream, this->gpuVaForMiFlush, this->rootDeviceEnvironment);
        this->osContext.setTlbFlushed(tlbFlushCounter);
    }
}

template <typename GfxFamily, typename Dispatcher>
size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeNewResourceHandler() {
    // Overestimate to avoid race
    return Dispatcher::getSizeTlbFlush(this->rootDeviceEnvironment);
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::isNewResourceHandleNeeded() {
    auto newResourcesBound = this->osContext.isTlbFlushRequired();

    if (debugManager.flags.DirectSubmissionNewResourceTlbFlush.get() != -1) {
        newResourcesBound = debugManager.flags.DirectSubmissionNewResourceTlbFlush.get();
    }

    return newResourcesBound;
}

template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::switchRingBuffersNeeded(size_t size, ResidencyContainer *allocationsForResidency) {
    if (this->ringCommandStream.getAvailableSpace() < size) {
        this->switchRingBuffers(allocationsForResidency);
    }
}

template <typename GfxFamily, typename Dispatcher>
inline uint64_t DirectSubmissionHw<GfxFamily, Dispatcher>::switchRingBuffers(ResidencyContainer *allocationsForResidency) {
    GraphicsAllocation *nextRingBuffer = switchRingBuffersAllocations(allocationsForResidency);

    this->handleRingRestartForUllsLightResidency(allocationsForResidency);

    uint64_t currentBufferGpuVa = ringCommandStream.getCurrentGpuAddressPosition();

    if (ringStart) {
        dispatchSwitchRingBufferSection(nextRingBuffer->getGpuAddress());
    }

    ringCommandStream.replaceBuffer(nextRingBuffer->getUnderlyingBuffer(), ringCommandStream.getMaxAvailableSpace());
    ringCommandStream.replaceGraphicsAllocation(nextRingBuffer);

    handleSwitchRingBuffers(allocationsForResidency);

    return currentBufferGpuVa;
}

template <typename GfxFamily, typename Dispatcher>
GraphicsAllocation *DirectSubmissionHw<GfxFamily, Dispatcher>::allocateRingBuffer() {
    bool isMultiOsContextCapable = osContext.getNumSupportedDevices() > 1u;
    const auto allocationSize = alignUp(minimumRingRequiredSize + additionalRingAllocationSize, MemoryConstants::pageSize64k);
    AllocationProperties commandStreamAllocationProperties{rootDeviceIndex,
                                                           true, allocationSize,
                                                           AllocationType::ringBuffer,
                                                           isMultiOsContextCapable, false, osContext.getDeviceBitfield()};
    return memoryManager->allocateGraphicsMemoryWithProperties(commandStreamAllocationProperties);
}

template <typename GfxFamily, typename Dispatcher>
GraphicsAllocation *DirectSubmissionHw<GfxFamily, Dispatcher>::fetchAsyncRingBuffer(bool blockingWait) {
    UNRECOVERABLE_IF(!this->asyncNewRingBufferAllocation.valid());
    if (blockingWait || this->asyncNewRingBufferAllocation.wait_for(std::chrono::microseconds(0)) != std::future_status::timeout) {
        auto alloc = this->asyncNewRingBufferAllocation.get();
        this->memoryOperationHandler->waitForAsyncResidency(&this->osContext, alloc);
        return alloc;
    }
    return nullptr;
}

template <typename GfxFamily, typename Dispatcher>
inline GraphicsAllocation *DirectSubmissionHw<GfxFamily, Dispatcher>::switchRingBuffersAllocations(ResidencyContainer *allocationsForResidency) {
    this->previousRingBuffer = this->currentRingBuffer;
    GraphicsAllocation *nextAllocation = nullptr;
    auto availableRingBuffersCount = 0u;

    auto addNewRingBufferToContainer = [&](GraphicsAllocation *allocation) {
        this->currentRingBuffer = static_cast<uint32_t>(this->ringBuffers.size());
        this->ringBuffers.emplace_back(0ull, 0ull, allocation);
        this->handleResidencyContainerForUllsLightNewRingAllocation(allocationsForResidency);
    };

    if (this->asyncNewRingBufferAllocation.valid()) {
        availableRingBuffersCount++;
        nextAllocation = fetchAsyncRingBuffer(false);
        if (nextAllocation) {
            addNewRingBufferToContainer(nextAllocation);
        }
    }

    UNRECOVERABLE_IF(this->ringBuffers.empty());
    auto ringBuffersCount = this->ringBuffers.size();
    for (uint32_t ringBufferIndex = (this->previousRingBuffer + 1) % ringBuffersCount;
         ringBufferIndex != this->previousRingBuffer;
         ringBufferIndex = (ringBufferIndex + 1) % ringBuffersCount) {
        if (ringBufferIndex == this->currentRingBuffer) {
            continue;
        }
        if (this->isCompleted(ringBufferIndex)) {
            availableRingBuffersCount++;
            if (nextAllocation == nullptr) {
                this->currentRingBuffer = ringBufferIndex;
                nextAllocation = this->ringBuffers[ringBufferIndex].ringBuffer;
            }
        }

        if (availableRingBuffersCount > 1) {
            break;
        }
    }

    if (this->asyncNewRingBufferAllocation.valid() && nextAllocation == nullptr) {
        nextAllocation = fetchAsyncRingBuffer(true);
        UNRECOVERABLE_IF(nextAllocation == nullptr);
        addNewRingBufferToContainer(nextAllocation);
    }

    if (nextAllocation == nullptr) {
        if (this->ringBuffers.size() == this->maxRingBufferCount) {
            this->currentRingBuffer = (this->currentRingBuffer + 1) % this->ringBuffers.size();
            nextAllocation = this->ringBuffers[this->currentRingBuffer].ringBuffer;
        } else {
            nextAllocation = allocateRingBuffer();
            auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef<GraphicsAllocation *>(&nextAllocation, 1u), false, false, false) == MemoryOperationsStatus::success;
            UNRECOVERABLE_IF(!ret);
            addNewRingBufferToContainer(nextAllocation);
        }
    }

    if (availableRingBuffersCount == 1 && this->ringBuffers.size() < this->maxRingBufferCount) {
        UNRECOVERABLE_IF(this->asyncNewRingBufferAllocation.valid());
        this->asyncNewRingBufferAllocation = std::async(getAsyncLaunchPolicy(), [this]() {
            auto asyncAllocation = this->allocateRingBuffer();
            UNRECOVERABLE_IF(asyncAllocation == nullptr);
            auto ret = memoryOperationHandler->makeResidentAsync(&this->osContext, asyncAllocation) == MemoryOperationsStatus::success;
            UNRECOVERABLE_IF(!ret);
            return asyncAllocation;
        });
    }

    UNRECOVERABLE_IF(this->currentRingBuffer == this->previousRingBuffer);
    return nextAllocation;
}

template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchMonitorFenceRequired(bool requireMonitorFence) {
    return false;
}

template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::deallocateResources() {
    if (this->asyncNewRingBufferAllocation.valid()) {
        auto asyncAllocation = fetchAsyncRingBuffer(true);
        memoryManager->freeGraphicsMemory(asyncAllocation);
    }
    for (uint32_t ringBufferIndex = 0; ringBufferIndex < this->ringBuffers.size(); ringBufferIndex++) {
        memoryManager->freeGraphicsMemory(this->ringBuffers[ringBufferIndex].ringBuffer);
    }
    this->ringBuffers.clear();
    if (semaphores) {
        memoryManager->freeGraphicsMemory(semaphores);
        semaphores = nullptr;
    }

    memoryManager->freeGraphicsMemory(deferredTasksListAllocation);
    memoryManager->freeGraphicsMemory(relaxedOrderingSchedulerAllocation);
}

template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchSystemMemoryFenceAddress() {
    this->makeGlobalFenceAlwaysResident();
    EncodeMemoryFence<GfxFamily>::encodeSystemMemoryFence(ringCommandStream, this->globalFenceAllocation);
}

template <typename GfxFamily, typename Dispatcher>
size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeSystemMemoryFenceAddress() {
    return EncodeMemoryFence<GfxFamily>::getSystemMemoryFenceSize();
}

template <typename GfxFamily, typename Dispatcher>
uint32_t DirectSubmissionHw<GfxFamily, Dispatcher>::getDispatchErrorCode() {
    return dispatchErrorCode;
}

template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchSemaphoreForPagingFence(uint64_t value) {
    using COMPARE_OPERATION = typename GfxFamily::MI_SEMAPHORE_WAIT::COMPARE_OPERATION;
    EncodeSemaphore<GfxFamily>::addMiSemaphoreWaitCommand(ringCommandStream,
                                                          this->gpuVaForPagingFenceSemaphore,
                                                          value,
                                                          COMPARE_OPERATION::COMPARE_OPERATION_SAD_GREATER_THAN_OR_EQUAL_SDD, false, false, false, false, this->useSemaphore64bCmd, nullptr);
}

template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeSemaphoreForPagingFence() {
    return EncodeSemaphore<GfxFamily>::getSizeMiSemaphoreWait();
}

} // namespace NEO
