http server: fix race condition for server->closed

Author: gdamore

src/supplemental/http/http_server.c

@@ -49,13 +49,13 @@ typedef struct http_sconn {
 	nni_http_handler *handler; // set if we deferred to read body
 	nni_http_handler *release; // set if we dispatched handler
 	bool              close;
-	bool              closed;
 	bool              finished;
 	nni_aio          *cbaio;
 	nni_aio          *rxaio;
 	nni_aio          *txaio;
 	nni_aio          *txdataio;
 	nni_reap_node     reap;
+	nni_atomic_flag   closed;
 } http_sconn;
 
 typedef struct http_error {
@@ -310,16 +310,15 @@ http_sc_reap(void *arg)
 }
 
 static void
-http_sc_close_locked(http_sconn *sc)
+http_sconn_close(http_sconn *sc)
 {
 	nni_http_conn *conn;
 
-	if (sc->closed) {
+	if (nni_atomic_flag_test_and_set(&sc->closed)) {
 		return;
 	}
 	NNI_ASSERT(!sc->finished);
 
-	sc->closed = true;
 	nni_aio_close(sc->rxaio);
 	nni_aio_close(sc->txaio);
 	nni_aio_close(sc->txdataio);
@@ -331,17 +330,6 @@ http_sc_close_locked(http_sconn *sc)
 	nni_reap(&http_sc_reap_list, sc);
 }
 
-static void
-http_sconn_close(http_sconn *sc)
-{
-	nni_http_server *s;
-	s = sc->server;
-
-	nni_mtx_lock(&s->mtx);
-	http_sc_close_locked(sc);
-	nni_mtx_unlock(&s->mtx);
-}
-
 static void
 http_sconn_txdatdone(void *arg)
 {
@@ -987,13 +975,16 @@ nni_http_server_init(nni_http_server **serverp, const nng_url *url)
 
 	nni_mtx_lock(&http_servers_lk);
 	NNI_LIST_FOREACH (&http_servers, s) {
+		nni_mtx_lock(&s->mtx);
 		if ((!s->closed) && (url->u_port == s->port) &&
 		    (strcmp(url->u_hostname, s->hostname) == 0)) {
 			*serverp = s;
 			s->refcnt++;
+			nni_mtx_unlock(&s->mtx);
 			nni_mtx_unlock(&http_servers_lk);
 			return (0);
 		}
+		nni_mtx_unlock(&s->mtx);
 	}
 
 	// We didn't find a server, try to make a new one.
@@ -1065,7 +1056,7 @@ http_server_stop(nni_http_server *s)
 	// Stopping the server is a hard stop -- it aborts any work
 	// being done by clients.  (No graceful shutdown).
 	NNI_LIST_FOREACH (&s->conns, sc) {
-		http_sc_close_locked(sc);
+		http_sconn_close(sc);
 	}
 }