Skip to content

Commit

Permalink
Move trimResidency functions to WddmResidencyController
Browse files Browse the repository at this point in the history 
Change-Id: I046fd34d5336b767ed38eda31e58e4a35ceee5f8
Signed-off-by: Maciej Dziuban <maciej.dziuban@intel.com>
  • Loading branch information
@DziubanMaciejIntel @Compute-Runtime-Automation
DziubanMaciejIntel authored and Compute-Runtime-Automation committed Oct 29, 2018
1 parent a30c70d commit dc4de3c
Show file tree
Hide file tree
Showing 17 changed files with 621 additions and 573 deletions.
2 changes: 1 addition & 1 deletion runtime/os_interface/windows/os_context_win.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ OsContextWin::OsContextImpl(Wddm &wddm, uint32_t osContextId) : wddm(wddm), resi
return;
}
}
initialized = wddmInterface->createMonitoredFence(*this);
initialized = wddmInterface->createMonitoredFence(this->residencyController);
};
OsContextWin::~OsContextImpl() {
wddm.getWddmInterface()->destroyHwQueue(hwQueueHandle);
Expand Down
6 changes: 3 additions & 3 deletions runtime/os_interface/windows/wddm/wddm.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -768,13 +768,13 @@ bool Wddm::waitOnGPU(D3DKMT_HANDLE context) {
return status == STATUS_SUCCESS;
}

bool Wddm::waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext) {
bool Wddm::waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence) {
NTSTATUS status = STATUS_SUCCESS;

if (lastFenceValue > *osContext.getResidencyController().getMonitoredFence().cpuAddress) {
if (lastFenceValue > *monitoredFence.cpuAddress) {
D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU waitFromCpu = {0};
waitFromCpu.ObjectCount = 1;
waitFromCpu.ObjectHandleArray = &osContext.getResidencyController().getMonitoredFence().fenceHandle;
waitFromCpu.ObjectHandleArray = &monitoredFence.fenceHandle;
waitFromCpu.FenceValueArray = &lastFenceValue;
waitFromCpu.hDevice = device;
waitFromCpu.hAsyncEvent = NULL;
Expand Down
2 changes: 1 addition & 1 deletion runtime/os_interface/windows/wddm/wddm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class Wddm {
MOCKABLE_VIRTUAL bool queryAdapterInfo();

MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, OsContextWin &osContext);
MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext);
MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence);

NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand);
VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmMemoryManager *memoryManager);
Expand Down
8 changes: 4 additions & 4 deletions runtime/os_interface/windows/wddm/wddm_interface.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ bool WddmInterface20::createHwQueue(PreemptionMode preemptionMode, OsContextWin
}
void WddmInterface20::destroyHwQueue(D3DKMT_HANDLE hwQueue) {}

bool WddmInterface::createMonitoredFence(OsContextWin &osContext) {
bool WddmInterface::createMonitoredFence(WddmResidencyController &residencyController) {
NTSTATUS Status;
D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject = {0};
CreateSynchronizationObject.hDevice = wddm.getDevice();
Expand All @@ -29,9 +29,9 @@ bool WddmInterface::createMonitoredFence(OsContextWin &osContext) {

DEBUG_BREAK_IF(STATUS_SUCCESS != Status);

osContext.getResidencyController().resetMonitoredFenceParams(CreateSynchronizationObject.hSyncObject,
reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress),
CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress);
residencyController.resetMonitoredFenceParams(CreateSynchronizationObject.hSyncObject,
reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress),
CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress);

return Status == STATUS_SUCCESS;
}
Expand Down
3 changes: 2 additions & 1 deletion runtime/os_interface/windows/wddm/wddm_interface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
namespace OCLRT {
class Gdi;
class Wddm;
class WddmResidencyController;

using OsContextWin = OsContext::OsContextImpl;

Expand All @@ -25,7 +26,7 @@ class WddmInterface {
WddmInterface() = delete;
virtual bool createHwQueue(PreemptionMode preemptionMode, OsContextWin &osContext) = 0;
virtual void destroyHwQueue(D3DKMT_HANDLE hwQueue) = 0;
bool createMonitoredFence(OsContextWin &osContext);
bool createMonitoredFence(WddmResidencyController &residencyController);
virtual const bool hwQueuesSupported() = 0;
virtual bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, OsContextWin &osContext) = 0;
Wddm &wddm;
Expand Down
2 changes: 1 addition & 1 deletion runtime/os_interface/windows/wddm_device_command_stream.inl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ MemoryManager *WddmCommandStreamReceiver<GfxFamily>::createMemoryManager(bool en

template <typename GfxFamily>
bool WddmCommandStreamReceiver<GfxFamily>::waitForFlushStamp(FlushStamp &flushStampToWait, OsContext &osContext) {
return wddm->waitFromCpu(flushStampToWait, *osContext.get());
return wddm->waitFromCpu(flushStampToWait, osContext.get()->getResidencyController().getMonitoredFence());
}

template <typename GfxFamily>
Expand Down
150 changes: 2 additions & 148 deletions runtime/os_interface/windows/wddm_memory_manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ void APIENTRY WddmMemoryManager::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *t
}

wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().acquireTrimCallbackLock();
wddmMemMngr->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().releaseTrimCallbackLock();
}

Expand Down Expand Up @@ -526,7 +526,7 @@ bool WddmMemoryManager::makeResidentResidencyAllocations(ResidencyContainer &all
uint64_t bytesToTrim = 0;
while ((result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) {
this->memoryBudgetExhausted = true;
bool trimmingDone = trimResidencyToBudget(bytesToTrim);
bool trimmingDone = this->getRegisteredOsContext(0u)->get()->getResidencyController().trimResidencyToBudget(bytesToTrim);
bool cantTrimFurther = !trimmingDone;
if (cantTrimFurther) {
result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim);
Expand Down Expand Up @@ -571,152 +571,6 @@ void WddmMemoryManager::makeNonResidentEvictionAllocations(ResidencyContainer &e
osContext.get()->getResidencyController().releaseLock();
}

void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) {
OsContext &osContext = *getRegisteredOsContext(0);
if (flags.PeriodicTrim) {
bool periodicTrimDone = false;
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
uint64_t sizeToTrim = 0;

osContext.get()->getResidencyController().acquireLock();

WddmAllocation *wddmAllocation = nullptr;
while ((wddmAllocation = osContext.get()->getResidencyController().getTrimCandidateHead()) != nullptr) {

DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", osContext.get()->getResidencyController().getLastTrimFenceValue());

// allocation was not used from last periodic trim
if (wddmAllocation->getResidencyData().getFenceValueForContextId(0) <= osContext.get()->getResidencyController().getLastTrimFenceValue()) {

DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(0));

uint32_t fragmentsToEvict = 0;

if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(0));
wddm->evict(&wddmAllocation->handle, 1, sizeToTrim);
}

for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= osContext.get()->getResidencyController().getLastTrimFenceValue()) {

DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0));

fragmentEvictHandles[fragmentsToEvict++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident = false;
}
}

if (fragmentsToEvict != 0) {
wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
}

wddmAllocation->getResidencyData().resident = false;

osContext.get()->getResidencyController().removeFromTrimCandidateList(wddmAllocation, false);
} else {
periodicTrimDone = true;
break;
}
}

if (osContext.get()->getResidencyController().checkTrimCandidateListCompaction()) {
osContext.get()->getResidencyController().compactTrimCandidateList();
}

osContext.get()->getResidencyController().releaseLock();
}

if (flags.TrimToBudget) {

osContext.get()->getResidencyController().acquireLock();

trimResidencyToBudget(bytes);

osContext.get()->getResidencyController().releaseLock();
}

if (flags.PeriodicTrim || flags.RestartPeriodicTrim) {
const auto newPeriodicTrimFenceValue = *osContext.get()->getResidencyController().getMonitoredFence().cpuAddress;
osContext.get()->getResidencyController().setLastTrimFenceValue(newPeriodicTrimFenceValue);
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", newPeriodicTrimFenceValue);
}
}

bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) {
bool trimToBudgetDone = false;
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
uint64_t numberOfBytesToTrim = bytes;
WddmAllocation *wddmAllocation = nullptr;
auto &osContext = *getRegisteredOsContext(0);

trimToBudgetDone = (numberOfBytesToTrim == 0);

while (!trimToBudgetDone) {
uint64_t lastFence = 0;
wddmAllocation = osContext.get()->getResidencyController().getTrimCandidateHead();

if (wddmAllocation == nullptr) {
break;
}

lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(0);
auto &monitoredFence = osContext.get()->getResidencyController().getMonitoredFence();

if (lastFence <= monitoredFence.lastSubmittedFence) {
uint32_t fragmentsToEvict = 0;
uint64_t sizeEvicted = 0;
uint64_t sizeToTrim = 0;

if (lastFence > *monitoredFence.cpuAddress) {
wddm->waitFromCpu(lastFence, *osContext.get());
}

if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
wddm->evict(&wddmAllocation->handle, 1, sizeToTrim);

sizeEvicted = wddmAllocation->getAlignedSize();
} else {
auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData;
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= monitoredFence.lastSubmittedFence) {
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData[allocationId].osHandleStorage->handle;
}
}

if (fragmentsToEvict != 0) {
wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);

for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= monitoredFence.lastSubmittedFence) {
fragmentStorageData[allocationId].residency->resident = false;
sizeEvicted += fragmentStorageData[allocationId].fragmentSize;
}
}
}
}

if (sizeEvicted >= numberOfBytesToTrim) {
numberOfBytesToTrim = 0;
} else {
numberOfBytesToTrim -= sizeEvicted;
}

wddmAllocation->getResidencyData().resident = false;
osContext.get()->getResidencyController().removeFromTrimCandidateList(wddmAllocation, false);
trimToBudgetDone = (numberOfBytesToTrim == 0);
} else {
trimToBudgetDone = true;
}
}

if (bytes > numberOfBytesToTrim && osContext.get()->getResidencyController().checkTrimCandidateListCompaction()) {
osContext.get()->getResidencyController().compactTrimCandidateList();
}

return numberOfBytesToTrim == 0;
}

bool WddmMemoryManager::mapAuxGpuVA(GraphicsAllocation *graphicsAllocation) {
return wddm->updateAuxTable(graphicsAllocation->getGpuAddress(), graphicsAllocation->gmm, true);
}
Expand Down
2 changes: 0 additions & 2 deletions runtime/os_interface/windows/wddm_memory_manager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,6 @@ class WddmMemoryManager : public MemoryManager {
AlignedMallocRestrictions *getAlignedMallocRestrictions() override;

protected:
void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes);
bool trimResidencyToBudget(uint64_t bytes);
VOID *trimCallbackHandle = nullptr;

GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle);
Expand Down
Loading

0 comments on commit dc4de3c

Please sign in to comment.