Skip to content
This repository has been archived by the owner on Jan 23, 2023. It is now read-only.

Commit

Permalink
Cache pad threadpool workitem queues (false sharing)
Browse files Browse the repository at this point in the history
  • Loading branch information
@benaadams
benaadams committed Jun 23, 2016
1 parent 63796e5 commit 498504c
Showing 1 changed file with 34 additions and 25 deletions.
59 changes: 34 additions & 25 deletions src/mscorlib/src/System/Threading/ThreadPool.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,19 @@ internal void Remove(T e)
}
}

// Enusre full cache line occupied so no false sharing with other work items
[StructLayout(LayoutKind.Explicit, Size = 64)]
internal struct PaddedWorkItem
{
[FieldOffset(0)]
public IThreadPoolWorkItem Item;
}


internal class WorkStealingQueue
{
private const int INITIAL_SIZE = 32;
internal volatile IThreadPoolWorkItem[] m_array = new IThreadPoolWorkItem[INITIAL_SIZE];
internal volatile PaddedWorkItem[] m_array = new PaddedWorkItem[INITIAL_SIZE];
private volatile int m_mask = INITIAL_SIZE - 1;

#if DEBUG
Expand Down Expand Up @@ -186,7 +195,7 @@ public void LocalPush(IThreadPoolWorkItem obj)
// When there are at least 2 elements' worth of space, we can take the fast path.
if (tail < m_headIndex + m_mask)
{
Volatile.Write(ref m_array[tail & m_mask], obj);
Volatile.Write(ref m_array[tail & m_mask].Item, obj);
m_tailIndex = tail + 1;
}
else
Expand All @@ -204,9 +213,9 @@ public void LocalPush(IThreadPoolWorkItem obj)
if (count >= m_mask)
{
// We're full; expand the queue by doubling its size.
IThreadPoolWorkItem[] newArray = new IThreadPoolWorkItem[m_array.Length << 1];
PaddedWorkItem[] newArray = new PaddedWorkItem[m_array.Length << 1];
for (int i = 0; i < m_array.Length; i++)
newArray[i] = m_array[(i + head) & m_mask];
newArray[i].Item = m_array[(i + head) & m_mask].Item;

// Reset the field values, incl. the mask.
m_array = newArray;
Expand All @@ -215,7 +224,7 @@ public void LocalPush(IThreadPoolWorkItem obj)
m_mask = (m_mask << 1) | 1;
}

Volatile.Write(ref m_array[tail & m_mask], obj);
Volatile.Write(ref m_array[tail & m_mask].Item, obj);
m_tailIndex = tail + 1;
}
finally
Expand All @@ -230,7 +239,7 @@ public void LocalPush(IThreadPoolWorkItem obj)
public bool LocalFindAndPop(IThreadPoolWorkItem obj)
{
// Fast path: check the tail. If equal, we can skip the lock.
if (m_array[(m_tailIndex - 1) & m_mask] == obj)
if (m_array[(m_tailIndex - 1) & m_mask].Item == obj)
{
IThreadPoolWorkItem unused;
if (LocalPop(out unused))
Expand All @@ -251,7 +260,7 @@ public bool LocalFindAndPop(IThreadPoolWorkItem obj)
// the work item, we are about to block anyway (which is very expensive).
for (int i = m_tailIndex - 2; i >= m_headIndex; i--)
{
if (m_array[i & m_mask] == obj)
if (m_array[i & m_mask].Item == obj)
{
// If we found the element, block out steals to avoid interference.
bool lockTaken = false;
Expand All @@ -260,11 +269,11 @@ public bool LocalFindAndPop(IThreadPoolWorkItem obj)
m_foreignLock.Enter(ref lockTaken);

// If we encountered a race condition, bail.
if (m_array[i & m_mask] == null)
if (m_array[i & m_mask].Item == null)
return false;

// Otherwise, null out the element.
Volatile.Write(ref m_array[i & m_mask], null);
Volatile.Write(ref m_array[i & m_mask].Item, null);

// And then check to see if we can fix up the indexes (if we're at
// the edge). If we can't, we just leave nulls in the array and they'll
Expand Down Expand Up @@ -307,12 +316,12 @@ public bool LocalPop(out IThreadPoolWorkItem obj)
if (m_headIndex <= tail)
{
int idx = tail & m_mask;
obj = Volatile.Read(ref m_array[idx]);
obj = Volatile.Read(ref m_array[idx].Item);

// Check for nulls in the array.
if (obj == null) continue;

m_array[idx] = null;
m_array[idx].Item = null;
return true;
}
else
Expand All @@ -327,12 +336,12 @@ public bool LocalPop(out IThreadPoolWorkItem obj)
{
// Element still available. Take it.
int idx = tail & m_mask;
obj = Volatile.Read(ref m_array[idx]);
obj = Volatile.Read(ref m_array[idx].Item);

// Check for nulls in the array.
if (obj == null) continue;

m_array[idx] = null;
m_array[idx].Item = null;
return true;
}
else
Expand Down Expand Up @@ -379,12 +388,12 @@ private bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal, int mil
if (head < m_tailIndex)
{
int idx = head & m_mask;
obj = Volatile.Read(ref m_array[idx]);
obj = Volatile.Read(ref m_array[idx].Item);

// Check for nulls in the array.
if (obj == null) continue;

m_array[idx] = null;
m_array[idx].Item = null;
return true;
}
else
Expand Down Expand Up @@ -414,7 +423,7 @@ private bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal, int mil
internal class QueueSegment
{
// Holds a segment of the queue. Enqueues/Dequeues start at element 0, and work their way up.
internal readonly IThreadPoolWorkItem[] nodes;
internal readonly PaddedWorkItem[] nodes;
private const int QueueSegmentLength = 256;

// Holds the indexes of the lowest and highest valid elements of the nodes array.
Expand Down Expand Up @@ -464,7 +473,7 @@ bool CompareExchangeIndexes(ref int prevUpper, int newUpper, ref int prevLower,
public QueueSegment()
{
Contract.Assert(QueueSegmentLength <= SixteenBits);
nodes = new IThreadPoolWorkItem[QueueSegmentLength];
nodes = new PaddedWorkItem[QueueSegmentLength];
}


Expand Down Expand Up @@ -498,8 +507,8 @@ public bool TryEnqueue(IThreadPoolWorkItem node)

if (CompareExchangeIndexes(ref upper, upper + 1, ref lower, lower))
{
Contract.Assert(Volatile.Read(ref nodes[upper]) == null);
Volatile.Write(ref nodes[upper], node);
Contract.Assert(Volatile.Read(ref nodes[upper].Item) == null);
Volatile.Write(ref nodes[upper].Item, node);
return true;
}
}
Expand Down Expand Up @@ -529,11 +538,11 @@ public bool TryDequeue(out IThreadPoolWorkItem node)
// written the node reference to the array. We need to spin until
// it shows up.
SpinWait spinner = new SpinWait();
while ((node = Volatile.Read(ref nodes[lower])) == null)
while ((node = Volatile.Read(ref nodes[lower].Item)) == null)
spinner.SpinOnce();

// Null-out the reference so the object can be GC'd earlier.
nodes[lower] = null;
nodes[lower].Item = null;

return true;
}
Expand Down Expand Up @@ -1736,10 +1745,10 @@ internal static IEnumerable<IThreadPoolWorkItem> EnumerateQueuedWorkItems(Thread
{
if (wsq != null && wsq.m_array != null)
{
IThreadPoolWorkItem[] items = wsq.m_array;
ThreadPoolWorkQueue.PaddedWorkItem[] items = wsq.m_array;
for (int i = 0; i < items.Length; i++)
{
IThreadPoolWorkItem item = items[i];
IThreadPoolWorkItem item = items[i].Item;
if (item != null)
yield return item;
}
Expand All @@ -1754,10 +1763,10 @@ internal static IEnumerable<IThreadPoolWorkItem> EnumerateQueuedWorkItems(Thread
segment != null;
segment = segment.Next)
{
IThreadPoolWorkItem[] items = segment.nodes;
ThreadPoolWorkQueue.PaddedWorkItem[] items = segment.nodes;
for (int i = 0; i < items.Length; i++)
{
IThreadPoolWorkItem item = items[i];
IThreadPoolWorkItem item = items[i].Item;
if (item != null)
yield return item;
}
Expand Down

0 comments on commit 498504c

Please sign in to comment.