Skip to content

Commit

Permalink
wip/sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@mtfriesen
mtfriesen committed Dec 11, 2024
1 parent 6a570eb commit 8a724e8
Show file tree
Hide file tree
Showing 7 changed files with 186 additions and 206 deletions.
15 changes: 0 additions & 15 deletions published/external/xdp/framechecksum.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,4 @@ C_ASSERT(sizeof(XDP_FRAME_CHECKSUM) == 1);

#pragma warning(pop)

#define XDP_FRAME_EXTENSION_CHECKSUM_NAME L"ms_frame_checksum"
#define XDP_FRAME_EXTENSION_CHECKSUM_VERSION_1 1U

#include <xdp/extension.h>

inline
XDP_FRAME_CHECKSUM *
XdpGetChecksumExtension(
_In_ XDP_FRAME *Frame,
_In_ XDP_EXTENSION *Extension
)
{
return (XDP_FRAME_CHECKSUM *)XdpGetExtensionData(Frame, Extension);
}

EXTERN_C_END
105 changes: 74 additions & 31 deletions src/xdp/tx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -747,8 +747,6 @@ XdpTxQueueCreate(
XDP_TX_QUEUE_KEY Key;
XDP_TX_QUEUE *TxQueue = NULL;
NTSTATUS Status;
UINT32 BufferSize, FrameSize, FrameOffset, FrameCount, TxCompletionSize;
UINT8 BufferAlignment, FrameAlignment, TxCompletionAlignment;
XDP_EXTENSION_INFO ExtensionInfo;
DECLARE_UNICODE_STRING_SIZE(
Name, ARRAYSIZE("if_" MAXUINT32_STR "_queue_" MAXUINT32_STR "_rx"));
Expand Down Expand Up @@ -881,6 +879,63 @@ XdpTxQueueCreate(
TxQueue->FrameExtensionSet, XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_NAME);
}

Status =
XdpIfOpenInterfaceOffloadHandle(
XdpIfGetIfSetHandle(TxQueue->Binding), &TxQueue->Key.HookId,
&TxQueue->InterfaceOffloadHandle);
if (!NT_SUCCESS(Status)) {
goto Exit;
}

*NewTxQueue = TxQueue;
Status = STATUS_SUCCESS;

Exit:

TraceInfo(
TRACE_CORE,
"TxQueue=%p Hook={%!HOOK_LAYER!, %!HOOK_DIR!, %!HOOK_SUBLAYER!} Status=%!STATUS!",
TxQueue, HookId->Layer, HookId->Direction, HookId->SubLayer, Status);

if (!NT_SUCCESS(Status)) {
if (TxQueue != NULL) {
XdpTxQueueDereference(TxQueue);
}
}

return Status;
}

static
VOID
XdpTxQueueDeleteRings(
_In_ XDP_TX_QUEUE *TxQueue
)
{
ASSERT(TxQueue->State == XdpTxQueueStateCreated || TxQueue->State == XdpTxQueueStateDeleted);

if (TxQueue->CompletionRing != NULL) {
XdpRingFreeRing(TxQueue->CompletionRing);
}
if (TxQueue->FrameRing != NULL) {
XdpRingFreeRing(TxQueue->FrameRing);
}
}

static
NTSTATUS
XdpTxQueueActivate(
_In_ XDP_TX_QUEUE *TxQueue
)
{
NTSTATUS Status;
UINT32 BufferSize, FrameSize, FrameOffset, FrameCount, TxCompletionSize;
UINT8 BufferAlignment, FrameAlignment, TxCompletionAlignment;

TraceEnter(TRACE_CORE, "TxQueue=%p", TxQueue);

ASSERT(TxQueue->State == XdpTxQueueStateCreated);

Status =
XdpExtensionSetAssignLayout(
TxQueue->BufferExtensionSet, sizeof(XDP_BUFFER), __alignof(XDP_BUFFER),
Expand Down Expand Up @@ -949,29 +1004,28 @@ XdpTxQueueCreate(
}

Status =
XdpIfOpenInterfaceOffloadHandle(
XdpIfGetIfSetHandle(TxQueue->Binding), &TxQueue->Key.HookId,
&TxQueue->InterfaceOffloadHandle);
XdpIfActivateTxQueue(
TxQueue->Binding, TxQueue->InterfaceTxQueue,
(XDP_TX_QUEUE_HANDLE)&TxQueue->Dispatch,
(XDP_TX_QUEUE_CONFIG_ACTIVATE)&TxQueue->ConfigActivate);
if (!NT_SUCCESS(Status)) {
TraceError(
TRACE_CORE, "TxQueue=%p XdpIfActivateTxQueue failed Status=%!STATUS!",
TxQueue, Status);
goto Exit;
}

*NewTxQueue = TxQueue;
Status = STATUS_SUCCESS;
TxQueue->State = XdpTxQueueStateActive;
TraceInfo(TRACE_CORE, "TxQueue=%p Activated", TxQueue);

Exit:

TraceInfo(
TRACE_CORE,
"TxQueue=%p Hook={%!HOOK_LAYER!, %!HOOK_DIR!, %!HOOK_SUBLAYER!} Status=%!STATUS!",
TxQueue, HookId->Layer, HookId->Direction, HookId->SubLayer, Status);

if (!NT_SUCCESS(Status)) {
if (TxQueue != NULL) {
XdpTxQueueDereference(TxQueue);
}
XdpTxQueueDeleteRings(TxQueue);
}

TraceExitStatus(TRACE_CORE);

return Status;
}

Expand Down Expand Up @@ -1112,21 +1166,15 @@ XdpTxQueueAddDatapathClient(
UNREFERENCED_PARAMETER(TxClientType);

if (TxQueue->State == XdpTxQueueStateCreated) {
Status =
XdpIfActivateTxQueue(
TxQueue->Binding, TxQueue->InterfaceTxQueue,
(XDP_TX_QUEUE_HANDLE)&TxQueue->Dispatch,
(XDP_TX_QUEUE_CONFIG_ACTIVATE)&TxQueue->ConfigActivate);
Status = XdpTxQueueActivate(TxQueue);

if (!NT_SUCCESS(Status)) {
TraceError(
TRACE_CORE, "TxQueue=%p XdpIfActivateTxQueue failed Status=%!STATUS!",
TxQueue, Status);
goto Exit;
}

TxQueue->State = XdpTxQueueStateActive;
}

TxClientEntry->NotifyRoutine(TxClientEntry, XDP_TX_QUEUE_NOTIFICATION_ATTACH);

SyncParams.TxQueue = TxQueue;
SyncParams.TxClientEntry = TxClientEntry;
XdpTxQueueSync(TxQueue, XdpTxQueueSyncAddDatapathClient, &SyncParams);
Expand Down Expand Up @@ -1273,12 +1321,7 @@ XdpTxQueueDelete(

TxQueue->State = XdpTxQueueStateDeleted;

if (TxQueue->CompletionRing != NULL) {
XdpRingFreeRing(TxQueue->CompletionRing);
}
if (TxQueue->FrameRing != NULL) {
XdpRingFreeRing(TxQueue->FrameRing);
}
XdpTxQueueDeleteRings(TxQueue);
if (TxQueue->TxFrameCompletionExtensionSet != NULL) {
XdpExtensionSetCleanup(TxQueue->TxFrameCompletionExtensionSet);
}
Expand Down
1 change: 1 addition & 0 deletions src/xdp/tx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ XdpTxQueueDereference(
typedef struct _XDP_TX_QUEUE_NOTIFY_ENTRY XDP_TX_QUEUE_NOTIFICATION_ENTRY;

typedef enum _XDP_TX_QUEUE_NOTIFICATION_TYPE {
XDP_TX_QUEUE_NOTIFICATION_ATTACH,
XDP_TX_QUEUE_NOTIFICATION_DETACH,
} XDP_TX_QUEUE_NOTIFICATION_TYPE;

Expand Down
145 changes: 72 additions & 73 deletions src/xdp/xsk.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2140,28 +2140,80 @@ XskNotifyTxQueue(
XSK *Xsk = CONTAINING_RECORD(NotificationEntry, XSK, Tx.Xdp.QueueNotificationEntry);
KIRQL OldIrql;

if (NotificationType != XDP_TX_QUEUE_NOTIFICATION_DETACH) {
return;
}
if (NotificationType == XDP_TX_QUEUE_NOTIFICATION_ATTACH) {
XDP_TX_QUEUE_CONFIG_ACTIVATE Config = XdpTxQueueGetConfig(Xsk->Tx.Xdp.Queue);;
XDP_EXTENSION_INFO ExtensionInfo;

//
// Set the state to detached, except when socket closure has raced this
// detach event.
//
KeAcquireSpinLock(&Xsk->Lock, &OldIrql);
if (Xsk->State != XskClosing) {
ASSERT(Xsk->State >= XskBinding && Xsk->State <= XskActive);
Xsk->State = XskDetached;
}
KeReleaseSpinLock(&Xsk->Lock, OldIrql);
ASSERT(Xsk->State >= XskActivating);

//
// This detach event is executing in the context of XDP binding work queue
// processing on the TX queue, so it is synchronized with other detach
// instances that also utilize the XDP binding work queue (detach during
// bind failure and detach during socket closure).
//
XskDetachTxIf(Xsk);
Xsk->Tx.Xdp.Flags.OutOfOrderCompletion = XdpTxQueueIsOutOfOrderCompletionEnabled(Config);
Xsk->Tx.Xdp.Flags.CompletionContext = XdpTxQueueIsTxCompletionContextEnabled(Config);

Xsk->Tx.Xdp.FrameRing = XdpTxQueueGetFrameRing(Config);

if (Xsk->Tx.Xdp.Flags.CompletionContext) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_NAME,
XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_VERSION_1,
XDP_EXTENSION_TYPE_FRAME);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.FrameTxCompletionExtension);
}

if (Xsk->Tx.Xdp.Flags.OutOfOrderCompletion) {
Xsk->Tx.Xdp.CompletionRing = XdpTxQueueGetCompletionRing(Config);

if (Xsk->Tx.Xdp.Flags.CompletionContext) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_NAME,
XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_VERSION_1,
XDP_EXTENSION_TYPE_TX_FRAME_COMPLETION);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.TxCompletionExtension);
}
}

Xsk->Tx.Xdp.Flags.VirtualAddressExt = XdpTxQueueIsVirtualAddressEnabled(Config);
if (Xsk->Tx.Xdp.Flags.VirtualAddressExt) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_BUFFER_EXTENSION_VIRTUAL_ADDRESS_NAME,
XDP_BUFFER_EXTENSION_VIRTUAL_ADDRESS_VERSION_1, XDP_EXTENSION_TYPE_BUFFER);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.VaExtension);
}

Xsk->Tx.Xdp.Flags.LogicalAddressExt = XdpTxQueueIsLogicalAddressEnabled(Config);
if (Xsk->Tx.Xdp.Flags.LogicalAddressExt) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_BUFFER_EXTENSION_LOGICAL_ADDRESS_NAME,
XDP_BUFFER_EXTENSION_LOGICAL_ADDRESS_VERSION_1, XDP_EXTENSION_TYPE_BUFFER);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.LaExtension);
}

Xsk->Tx.Xdp.Flags.MdlExt = XdpTxQueueIsMdlEnabled(Config);
if (Xsk->Tx.Xdp.Flags.MdlExt) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_BUFFER_EXTENSION_MDL_NAME,
XDP_BUFFER_EXTENSION_MDL_VERSION_1, XDP_EXTENSION_TYPE_BUFFER);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.MdlExtension);
}
} else if (NotificationType == XDP_TX_QUEUE_NOTIFICATION_DETACH) {
//
// Set the state to detached, except when socket closure has raced this
// detach event.
//
KeAcquireSpinLock(&Xsk->Lock, &OldIrql);
if (Xsk->State != XskClosing) {
ASSERT(Xsk->State >= XskBinding && Xsk->State <= XskActive);
Xsk->State = XskDetached;
}
KeReleaseSpinLock(&Xsk->Lock, OldIrql);

//
// This detach event is executing in the context of XDP binding work queue
// processing on the TX queue, so it is synchronized with other detach
// instances that also utilize the XDP binding work queue (detach during
// bind failure and detach during socket closure).
//
XskDetachTxIf(Xsk);
}
}

static
Expand Down Expand Up @@ -2268,9 +2320,7 @@ XskBindTxIf(
{
XSK_BINDING_WORKITEM *WorkItem = (XSK_BINDING_WORKITEM *)Item;
XSK *Xsk = WorkItem->Xsk;
XDP_TX_QUEUE_CONFIG_ACTIVATE Config;
const XDP_TX_CAPABILITIES *InterfaceCapabilities;
XDP_EXTENSION_INFO ExtensionInfo;
NTSTATUS Status;

TraceEnter(TRACE_XSK, "Xsk=%p", Xsk);
Expand All @@ -2292,57 +2342,6 @@ XskBindTxIf(
Xsk->Tx.Xdp.MaxBufferLength = InterfaceCapabilities->MaximumBufferSize;
Xsk->Tx.Xdp.MaxFrameLength = InterfaceCapabilities->MaximumFrameSize;

Config = XdpTxQueueGetConfig(Xsk->Tx.Xdp.Queue);

Xsk->Tx.Xdp.Flags.OutOfOrderCompletion = XdpTxQueueIsOutOfOrderCompletionEnabled(Config);
Xsk->Tx.Xdp.Flags.CompletionContext = XdpTxQueueIsTxCompletionContextEnabled(Config);

Xsk->Tx.Xdp.FrameRing = XdpTxQueueGetFrameRing(Config);

if (Xsk->Tx.Xdp.Flags.CompletionContext) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_NAME,
XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_VERSION_1,
XDP_EXTENSION_TYPE_FRAME);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.FrameTxCompletionExtension);
}

if (Xsk->Tx.Xdp.Flags.OutOfOrderCompletion) {
Xsk->Tx.Xdp.CompletionRing = XdpTxQueueGetCompletionRing(Config);

if (Xsk->Tx.Xdp.Flags.CompletionContext) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_NAME,
XDP_TX_FRAME_COMPLETION_CONTEXT_EXTENSION_VERSION_1,
XDP_EXTENSION_TYPE_TX_FRAME_COMPLETION);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.TxCompletionExtension);
}
}

Xsk->Tx.Xdp.Flags.VirtualAddressExt = XdpTxQueueIsVirtualAddressEnabled(Config);
if (Xsk->Tx.Xdp.Flags.VirtualAddressExt) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_BUFFER_EXTENSION_VIRTUAL_ADDRESS_NAME,
XDP_BUFFER_EXTENSION_VIRTUAL_ADDRESS_VERSION_1, XDP_EXTENSION_TYPE_BUFFER);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.VaExtension);
}

Xsk->Tx.Xdp.Flags.LogicalAddressExt = XdpTxQueueIsLogicalAddressEnabled(Config);
if (Xsk->Tx.Xdp.Flags.LogicalAddressExt) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_BUFFER_EXTENSION_LOGICAL_ADDRESS_NAME,
XDP_BUFFER_EXTENSION_LOGICAL_ADDRESS_VERSION_1, XDP_EXTENSION_TYPE_BUFFER);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.LaExtension);
}

Xsk->Tx.Xdp.Flags.MdlExt = XdpTxQueueIsMdlEnabled(Config);
if (Xsk->Tx.Xdp.Flags.MdlExt) {
XdpInitializeExtensionInfo(
&ExtensionInfo, XDP_BUFFER_EXTENSION_MDL_NAME,
XDP_BUFFER_EXTENSION_MDL_VERSION_1, XDP_EXTENSION_TYPE_BUFFER);
XdpTxQueueGetExtension(Config, &ExtensionInfo, &Xsk->Tx.Xdp.MdlExtension);
}

Status = STATUS_SUCCESS;

Exit:
Expand Down
4 changes: 4 additions & 0 deletions src/xdplwf/precomp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,12 @@
#include <xdp/buffervirtualaddress.h>
#include <xdp/control.h>
#include <xdp/datapath.h>
#include <xdp/framechecksum.h>
#include <xdp/framechecksumextension.h>
#include <xdp/framefragment.h>
#include <xdp/frameinterfacecontext.h>
#include <xdp/framelayout.h>
#include <xdp/framelayoutextension.h>
#include <xdp/framerxaction.h>
#include <xdp/hookid.h>
#include <xdp/ndis6.h>
Expand Down
Loading

0 comments on commit 8a724e8

Please sign in to comment.