diff --git a/integration/test_aggregation_integration.py b/integration/test_aggregation_integration.py
index 9d224200..f08117d5 100644
--- a/integration/test_aggregation_integration.py
+++ b/integration/test_aggregation_integration.py
@@ -1094,9 +1094,9 @@ def test_aggregate_and_query_with_different_fixed_windows(setup_teardown_test, p
         {
             "col1": 10,
             "number_of_stuff_avg_1h": math.nan,
-            "number_of_stuff_max_1h": -math.inf,
-            "number_of_stuff_min_1h": math.inf,
-            "number_of_stuff_sum_1h": 0.0,
+            "number_of_stuff_max_1h": math.nan,
+            "number_of_stuff_min_1h": math.nan,
+            "number_of_stuff_sum_1h": math.nan,
             "time": datetime.datetime(2020, 7, 22, 2, 15, tzinfo=datetime.timezone.utc),
         },
     ]
diff --git a/storey/table.py b/storey/table.py
index 60d0f5c3..fca9146c 100644
--- a/storey/table.py
+++ b/storey/table.py
@@ -552,7 +552,11 @@ def __init__(
         windows = {}
         for aggregation_metadata in aggregates:
             for meta in aggregation_metadata.aggregations:
-                for aggr, is_hidden in get_all_raw_aggregates_with_hidden([meta]).items():
+                meta_aggregates = get_all_raw_aggregates_with_hidden([meta]).items()
+                if not any(ag[0] == "count" for ag in meta_aggregates):
+                    meta_aggregates = list(meta_aggregates)
+                    meta_aggregates.append(("count", True))
+                for aggr, is_hidden in meta_aggregates:
                     if (
                         aggregation_metadata.name,
                         aggr,
@@ -629,8 +633,13 @@ def aggregate(self, data, timestamp):
     def get_features(self, timestamp):
         result = {}
         for aggregation_bucket in self.aggregation_buckets.values():
-            if isinstance(aggregation_bucket, VirtualAggregationBuckets) or aggregation_bucket.explicit_windows:
+            if isinstance(aggregation_bucket, VirtualAggregationBuckets):
                 result.update(aggregation_bucket.get_features(timestamp))
+            elif aggregation_bucket.explicit_windows:
+                count_features = self.aggregation_buckets[f"{aggregation_bucket.name}_count"].get_features(
+                    timestamp, aggregation_bucket.explicit_windows.windows
+                )
+                result.update(aggregation_bucket.get_features(timestamp, count_features=count_features))
 
         return result
 
@@ -832,7 +841,7 @@ def get_aggregation_for_aggregation(self):
             return "sum"
         return self.aggregation
 
-    def get_features(self, timestamp, windows=None):
+    def get_features(self, timestamp, windows=None, count_features=None):
         result = {}
         if not windows:
             if self.explicit_windows:
@@ -852,7 +861,11 @@ def get_features(self, timestamp, windows=None):
         # In case our pre aggregates already have the answer
         for win in windows:
             result[f"{self.name}_{self.aggregation}_{win[1]}"] = self._current_aggregate_values[win].value
-
+            if (
+                self.aggregation != "count"
+                and (count_features.get(f"{self.name}_count_{win[1]}", 0) if count_features else 1) == 0
+            ):
+                result[f"{self.name}_{self.aggregation}_{win[1]}"] = math.nan
         return result
 
     def calculate_features(self, timestamp, windows):