diff --git a/src/Prometheus.Client/Gauge.cs b/src/Prometheus.Client/Gauge.cs
index 9ee70e2..b0da7aa 100644
--- a/src/Prometheus.Client/Gauge.cs
+++ b/src/Prometheus.Client/Gauge.cs
@@ -87,9 +87,6 @@ public void Set(double val)
 
             public void Set(double val, long? timestamp)
             {
-                if (double.IsNaN(val))
-                    return;
-
                 _value.Value = val;
                 TimestampIfRequired(timestamp);
             }
diff --git a/src/Prometheus.Client/ThreadSafeDouble.cs b/src/Prometheus.Client/ThreadSafeDouble.cs
index 57f5fe6..582f5e6 100644
--- a/src/Prometheus.Client/ThreadSafeDouble.cs
+++ b/src/Prometheus.Client/ThreadSafeDouble.cs
@@ -26,6 +26,9 @@ public void Add(double increment)
                 long initialValue = Interlocked.Read(ref _value);
                 double computedValue = BitConverter.Int64BitsToDouble(initialValue) + increment;
 
+                if (double.IsNaN(computedValue))
+                    throw new InvalidOperationException("Cannot increment the NaN value.");
+
                 if (initialValue == Interlocked.CompareExchange(ref _value, BitConverter.DoubleToInt64Bits(computedValue), initialValue))
                     return;
             }
diff --git a/tests/Prometheus.Client.Tests/GaugeTests.cs b/tests/Prometheus.Client.Tests/GaugeTests.cs
index 687f7a6..044893c 100644
--- a/tests/Prometheus.Client.Tests/GaugeTests.cs
+++ b/tests/Prometheus.Client.Tests/GaugeTests.cs
@@ -81,6 +81,9 @@ public async Task Collection()
             gauge2.Inc(1);
             gauge2.WithLabels("any", "2").Dec(5);
 
+            var nanGauge = factory.CreateGauge("nangauge", "example of NaN");
+            nanGauge.Set(double.NaN);
+
             string formattedText = null;
 
             using (var stream = new MemoryStream())
@@ -89,6 +92,7 @@ public async Task Collection()
                 {
                     ((ICollector)gauge).Collect(writer);
                     ((ICollector)gauge2).Collect(writer);
+                    ((ICollector)nanGauge).Collect(writer);
                     await writer.CloseWriterAsync();
                 }
 
@@ -264,5 +268,44 @@ public void WithoutLabels()
 
             Assert.Equal(2, gauge.Value);
         }
+
+        [Fact]
+        public void ShouldAllowNaN()
+        {
+            var registry = new CollectorRegistry();
+            var factory = new MetricFactory(registry);
+
+            var gauge = factory.CreateGauge("test_gauge", string.Empty);
+
+            gauge.Set(double.NaN);
+
+            Assert.Equal(double.NaN, gauge.Value);
+        }
+
+        [Fact]
+        public void ShouldThrowOnIncIfNaN()
+        {
+            var registry = new CollectorRegistry();
+            var factory = new MetricFactory(registry);
+
+            var gauge = factory.CreateGauge("test_gauge", string.Empty);
+
+            gauge.Set(double.NaN);
+
+            Assert.Throws<InvalidOperationException>(() => gauge.Inc());
+        }
+
+        [Fact]
+        public void ShouldThrowOnDecIfNaN()
+        {
+            var registry = new CollectorRegistry();
+            var factory = new MetricFactory(registry);
+
+            var gauge = factory.CreateGauge("test_gauge", string.Empty);
+
+            gauge.Set(double.NaN);
+
+            Assert.Throws<InvalidOperationException>(() => gauge.Dec());
+        }
     }
 }
diff --git a/tests/Prometheus.Client.Tests/Resources/GaugeTests_Collection.txt b/tests/Prometheus.Client.Tests/Resources/GaugeTests_Collection.txt
index 20d3e1c..992c742 100644
--- a/tests/Prometheus.Client.Tests/Resources/GaugeTests_Collection.txt
+++ b/tests/Prometheus.Client.Tests/Resources/GaugeTests_Collection.txt
@@ -6,3 +6,6 @@ test{category="some"} 5
 # TYPE nextgauge gauge
 nextgauge 1
 nextgauge{group="any",type="2"} -5
+# HELP nangauge example of NaN
+# TYPE nangauge gauge
+nangauge NaN