fixes #1619 expose expire threads tunables

This change makes expire threads tunable follows the same strategy as
taskq threads tunables.

Add NNG_NUM_EXPIRE_THREADS to override the default behavior (`n_cpu`
expire threads).

The NNG_MAX_EXPIRE_THREADS limit is always applied if > 0, even if you
specify the desired number of threads using NNG_NUM_EXPIRE_THREADS.

NNG_EXPIRE_THREADS is not used anymore. This was only referenced in the
code but never defined on CMake.

The logic to cap expire threads between 1 and 256 was removed. If users
set no limits, whatever value they choose will be used instead of being
silently overridden by us.

CMakeLists.txt

@@ -132,9 +132,7 @@ endif ()
 
 # Expire threads. This runs the timeout handling, and having more of them
 # reduces contention on the common locks used for aio expiration.
-# The default is to allocate up to max(8, ncpu).  The upper limit can be
-# overridden here.
-set(NNG_MAX_EXPIRE_THREADS 8 CACHE STRING "Upper bound on expire threads, between 1...256")
+set(NNG_MAX_EXPIRE_THREADS 8 CACHE STRING "Upper bound on expire threads, 0 for no limit")
 mark_as_advanced(NNG_MAX_EXPIRE_THREADS)
 if (NNG_MAX_EXPIRE_THREADS)
     add_definitions(-DNNG_MAX_EXPIRE_THREADS=${NNG_MAX_EXPIRE_THREADS})

src/core/aio.c

@@ -41,7 +41,7 @@ static int                nni_aio_expire_q_cnt;
 // condition variable, and expiration thread.  By default, this is one
 // per CPU core present -- the goal being to reduce overall pressure
 // caused by a single lock.  The number of queues (and threads) can
-// be tuned using the NNG_EXPIRE_THREADS tunable.
+// be tuned using the NNG_NUM_EXPIRE_THREADS tunable.
 //
 // We will not permit an AIO
 // to be marked done if an expiration is outstanding.
@@ -795,24 +795,16 @@ nni_aio_sys_init(void)
 {
 	int num_thr;
 
-	// We create a thread per CPU core for expiration by default.
+#ifndef NNG_NUM_EXPIRE_THREADS
 	num_thr = nni_plat_ncpu();
-#ifndef NNG_EXPIRE_THREADS
-#ifndef NNG_MAX_EXPIRE_THREADS
-#define NNG_MAX_EXPIRE_THREADS 8
+#else
+	num_thr = NNG_NUM_EXPIRE_THREADS;
 #endif
-	if ((num_thr > NNG_MAX_EXPIRE_THREADS) && (NNG_MAX_EXPIRE_THREADS > 0)) {
+#if NNG_MAX_EXPIRE_THREADS > 0
+	if (num_thr > NNG_MAX_EXPIRE_THREADS) {
 		num_thr = NNG_MAX_EXPIRE_THREADS;
 	}
-#else
-	num_thr = NNG_EXPIRE_THREADS;
 #endif
-	if (num_thr > 256) { // upper limits
-		num_thr = 256;
-	}
-	if (num_thr < 1) {
-		num_thr = 1;
-	}
 
 	nni_aio_expire_q_list =
 	    nni_zalloc(sizeof(nni_aio_expire_q *) * num_thr);