Consolidating metrics declaration in storage/seg crate into a single …

…module (#483)

* move all seg metrics into its own module, add item_allocate

* moving the last few

src/storage/seg/src/hashtable/mod.rs

@@ -82,29 +82,6 @@ mod hash_bucket;
 
 pub(crate) use hash_bucket::*;
 
-counter!(HASH_TAG_COLLISION, "number of partial hash collisions");
-counter!(HASH_INSERT, "number of inserts into the hash table");
-counter!(
-    HASH_INSERT_EX,
-    "number of hash table inserts which failed, likely due to capacity"
-);
-counter!(
-    HASH_REMOVE,
-    "number of hash table entries which have been removed"
-);
-counter!(
-    HASH_LOOKUP,
-    "total number of lookups against the hash table"
-);
-counter!(
-    ITEM_RELINK,
-    "number of times items have been relinked to different locations"
-);
-counter!(ITEM_REPLACE, "number of times items have been replaced");
-counter!(ITEM_DELETE, "number of items removed from the hash table");
-counter!(ITEM_EXPIRE, "number of items removed due to expiration");
-counter!(ITEM_EVICT, "number of items removed due to eviction");
-
 #[derive(Debug)]
 struct IterState {
     bucket_id: usize,

src/storage/seg/src/lib.rs

@@ -28,7 +28,6 @@ extern crate logger;
 
 // external crate includes
 use common::time::Seconds;
-use rustcommon_metrics::*;
 
 // includes from core/std
 use core::hash::{BuildHasher, Hasher};
@@ -44,6 +43,7 @@ mod error;
 mod eviction;
 mod hashtable;
 mod item;
+mod metrics;
 mod rand;
 mod seg;
 mod segments;
@@ -71,6 +71,7 @@ pub(crate) type Instant = common::time::Instant<Seconds<u32>>;
 pub(crate) use crate::rand::*;
 pub(crate) use hashtable::*;
 pub(crate) use item::*;
+pub(crate) use metrics::*;
 pub(crate) use segments::*;
 pub(crate) use ttl_buckets::*;
 

src/storage/seg/src/metrics.rs

@@ -0,0 +1,79 @@
+// Copyright 2022 Twitter, Inc.
+// Licensed under the Apache License, Version 2.0
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// All metrics for the Seg crate
+
+use rustcommon_metrics::*;
+
+// segment related
+counter!(SEGMENT_REQUEST, "number of segment allocation attempts");
+counter!(
+    SEGMENT_REQUEST_FAILURE,
+    "number of segment allocation attempts which failed"
+);
+counter!(
+    SEGMENT_REQUEST_SUCCESS,
+    "number of segment allocation attempts which were successful"
+);
+counter!(SEGMENT_EVICT, "number of segments evicted");
+counter!(
+    SEGMENT_EVICT_EX,
+    "number of exceptions while evicting segments"
+);
+counter!(
+    SEGMENT_RETURN,
+    "total number of segments returned to the free pool"
+);
+counter!(SEGMENT_MERGE, "total number of segments merged");
+counter!(SEGMENT_CLEAR, "number of segments cleared");
+counter!(SEGMENT_EXPIRE, "number of segments expired");
+counter!(
+    CLEAR_TIME,
+    "amount of time, in nanoseconds, spent clearing segments"
+);
+counter!(
+    EXPIRE_TIME,
+    "amount of time, in nanoseconds, spent expiring segments"
+);
+gauge!(EVICT_TIME, "time, in nanoseconds, spent evicting segments");
+gauge!(SEGMENT_FREE, "current number of free segments");
+gauge!(SEGMENT_CURRENT, "current number of segments");
+
+// hash table related
+counter!(HASH_TAG_COLLISION, "number of partial hash collisions");
+counter!(HASH_INSERT, "number of inserts into the hash table");
+counter!(
+    HASH_INSERT_EX,
+    "number of hash table inserts which failed, likely due to capacity"
+);
+counter!(
+    HASH_REMOVE,
+    "number of hash table entries which have been removed"
+);
+counter!(
+    HASH_LOOKUP,
+    "total number of lookups against the hash table"
+);
+counter!(
+    ITEM_RELINK,
+    "number of times items have been relinked to different locations"
+);
+
+// item related
+counter!(ITEM_ALLOCATE, "number of times items have been allocated");
+counter!(ITEM_REPLACE, "number of times items have been replaced");
+counter!(ITEM_DELETE, "number of items removed from the hash table");
+counter!(ITEM_EXPIRE, "number of items removed due to expiration");
+counter!(ITEM_EVICT, "number of items removed due to eviction");
+counter!(ITEM_COMPACTED, "number of items which have been compacted");
+gauge!(ITEM_CURRENT, "current number of live items");
+gauge!(
+    ITEM_CURRENT_BYTES,
+    "current number of live bytes for storing items"
+);
+gauge!(ITEM_DEAD, "current number of dead items");
+gauge!(
+    ITEM_DEAD_BYTES,
+    "current number of dead bytes for storing items"
+);

src/storage/seg/src/seg.rs

@@ -10,16 +10,6 @@ use std::cmp::min;
 
 const RESERVE_RETRIES: usize = 3;
 
-counter!(SEGMENT_REQUEST, "number of segment allocation attempts");
-counter!(
-    SEGMENT_REQUEST_FAILURE,
-    "number of segment allocation attempts which failed"
-);
-counter!(
-    SEGMENT_REQUEST_SUCCESS,
-    "number of segment allocation attempts which were successful"
-);
-
 /// A pre-allocated key-value store with eager expiration. It uses a
 /// segment-structured design that stores data in fixed-size segments, grouping
 /// objects with nearby expiration time into the same segment, and lifting most

src/storage/seg/src/segments/segment.rs

@@ -6,18 +6,6 @@ use super::{SegmentHeader, SegmentsError};
 use crate::*;
 use core::num::NonZeroU32;
 
-gauge!(ITEM_CURRENT, "current number of live items");
-gauge!(
-    ITEM_CURRENT_BYTES,
-    "current number of live bytes for storing items"
-);
-gauge!(ITEM_DEAD, "current number of dead items");
-gauge!(
-    ITEM_DEAD_BYTES,
-    "current number of dead bytes for storing items"
-);
-counter!(ITEM_COMPACTED, "number of items which have been compacted");
-
 pub const SEG_MAGIC: u64 = 0xBADC0FFEEBADCAFE;
 
 /// A `Segment` is a contiguous allocation of bytes and an associated header
@@ -289,6 +277,7 @@ impl<'a> Segment<'a> {
     pub(crate) fn alloc_item(&mut self, size: i32) -> RawItem {
         let offset = self.write_offset() as usize;
         self.incr_item(size);
+        ITEM_ALLOCATE.increment();
         ITEM_CURRENT.increment();
         ITEM_CURRENT_BYTES.add(size as _);
 

src/storage/seg/src/segments/segments.rs

@@ -4,25 +4,10 @@
 
 use crate::eviction::*;
 use crate::item::*;
-use crate::seg::{SEGMENT_REQUEST, SEGMENT_REQUEST_SUCCESS};
 use crate::segments::*;
 use core::num::NonZeroU32;
 use datapool::*;
 
-gauge!(EVICT_TIME, "time, in nanoseconds, spent evicting segments");
-counter!(SEGMENT_EVICT, "number of segments evicted");
-counter!(
-    SEGMENT_EVICT_EX,
-    "number of exceptions while evicting segments"
-);
-counter!(
-    SEGMENT_RETURN,
-    "total number of segments returned to the free pool"
-);
-gauge!(SEGMENT_FREE, "current number of free segments");
-counter!(SEGMENT_MERGE, "total number of segments merged");
-gauge!(SEGMENT_CURRENT, "current number of segments");
-
 /// `Segments` contain all items within the cache. This struct is a collection
 /// of individual `Segment`s which are represented by a `SegmentHeader` and a
 /// subslice of bytes from a contiguous heap allocation.

src/storage/seg/src/ttl_buckets/mod.rs

@@ -23,16 +23,3 @@ mod tests;
 pub use error::TtlBucketsError;
 pub use ttl_bucket::TtlBucket;
 pub use ttl_buckets::TtlBuckets;
-
-use rustcommon_metrics::*;
-
-counter!(SEGMENT_CLEAR, "number of segments cleared");
-counter!(SEGMENT_EXPIRE, "number of segments expired");
-counter!(
-    CLEAR_TIME,
-    "amount of time, in nanoseconds, spent clearing segments"
-);
-counter!(
-    EXPIRE_TIME,
-    "amount of time, in nanoseconds, spent expiring segments"
-);

src/storage/seg/src/ttl_buckets/ttl_bucket.rs

@@ -26,7 +26,6 @@
 //! └──────────────────────────────────────────────────────────┘
 //! ```
 
-use super::{SEGMENT_CLEAR, SEGMENT_EXPIRE};
 use crate::*;
 use core::num::NonZeroU32;
 

src/storage/seg/src/ttl_buckets/ttl_buckets.rs

@@ -20,7 +20,6 @@
 //! [Segcache paper](https://www.usenix.org/system/files/nsdi21-yang.pdf) for
 //! more detail.
 
-use super::{CLEAR_TIME, EXPIRE_TIME};
 use crate::*;
 
 const N_BUCKET_PER_STEP_N_BIT: usize = 8;