Skip to content

Commit

Permalink
remove window related functionality, this is now in the query service
Browse files Browse the repository at this point in the history
  • Loading branch information
@nomaxg
nomaxg committed Mar 12, 2024
1 parent 4ccde8a commit 4b90848
Show file tree
Hide file tree
Showing 6 changed files with 11 additions and 518 deletions.
61 changes: 1 addition & 60 deletions sequencer/api/availability.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,63 +2,4 @@
PATH = ["block/:height/namespace/:namespace"]
":height" = "Integer"
":namespace" = "Integer"
DOC = "Get the transactions in a namespace of the given block, along with a proof."

[route.gettimestampwindow]
PATH = [
"headers/window/:start/:end",
"headers/window/from/:height/:end",
"headers/window/from/hash/:hash/:end",
]
":start" = "Integer"
":end" = "Integer"
":height" = "Integer"
":hash" = "TaggedBase64"
DOC = """
Get block headers in a time window.
Returns all available headers, in order, whose timestamps fall between `:start` (inclusive) and
`:end` (exclusive), or between the block indicated by `:height` or `:hash` (inclusive) and `:end`
(exclusive). The response also includes one block before the desired window (unless the window
includes the genesis block) and one block after the window. This proves to the client that the
server has not omitted any blocks whose timestamps fall within the desired window.
It is possible that not all blocks in the desired window are available when this endpoint is called.
In that case, whichever blocks are available are included in the response, and `next` is `null` to
indicate that the response is not complete. The client can then use one of the `/from/` forms of
this endpoint to fetch the remaining blocks from where the first response left off, once they become
available. If no blocks are available, not even `prev`, this endpoint will return an error.
Returns
```json
{
"from": "integer", // block number of the first block in the window, unless the window is empty,
// then block number of `next`
"window": ["Header"],
"prev": "Header", // nullable
"next": "Header" // nullable
}
```
where a `Header` has the schema:
```json
{
"metadata": {
"timestamp": "integer",
"l1_head": "integer",
"l1_finalized": {
"timestamp": "integer",
"number": "integer",
"hash": "TaggedBase64"
},
}
"transactions_root": {
"root": ["integer"]
}
}
```
All timestamps are denominated in an integer number of seconds.
"""
DOC = "Get the transactions in a namespace of the given block, along with a proof."
181 changes: 1 addition & 180 deletions sequencer/src/api.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,10 +413,9 @@ mod api_tests {
Header, Transaction,
};
use async_compatibility_layer::logging::{setup_backtrace, setup_logging};
use async_std::task::sleep;
use commit::Committable;
use data_source::testing::TestableSequencerDataSource;
use endpoints::{NamespaceProofQueryData, TimeWindowQueryData};
use endpoints::NamespaceProofQueryData;
use futures::{
future::join_all,
stream::{StreamExt, TryStreamExt},
Expand All @@ -427,7 +426,6 @@ mod api_tests {
};
use hotshot_types::vid::vid_scheme;
use portpicker::pick_unused_port;
use std::time::Duration;
use surf_disco::Client;
use test_helpers::{
state_signature_test_helper, state_test_helper, status_test_helper, submit_test_helper,
Expand Down Expand Up @@ -636,183 +634,6 @@ mod api_tests {
.unwrap();
assert_eq!(chain, new_chain);
}

#[async_std::test]
pub(crate) async fn test_timestamp_window<D: TestableSequencerDataSource>() {
setup_logging();
setup_backtrace();

// Start query service.
let port = pick_unused_port().expect("No ports free");
let storage = D::create_storage().await;
let _network = TestNetwork::new(
D::options(&storage, options::Http { port }.into()).status(Default::default()),
)
.await;

// Connect client.
let client: Client<ServerError> =
Client::new(format!("http://localhost:{port}").parse().unwrap());
client.connect(None).await;

// Wait for blocks with at least three different timestamps to be sequenced. This lets us
// test all the edge cases.
let mut test_blocks: Vec<Vec<Header>> = vec![];
while test_blocks.len() < 3 {
let num_blocks = test_blocks.iter().flatten().count();

// Wait for the next block to be sequenced.
loop {
let block_height = client
.get::<usize>("status/block-height")
.send()
.await
.unwrap();
if block_height > num_blocks {
break;
}
tracing::info!("waiting for block {num_blocks}, current height {block_height}");
sleep(Duration::from_secs(1)).await;
}

let block: BlockQueryData<SeqTypes> = client
.get(&format!("availability/block/{num_blocks}"))
.send()
.await
.unwrap();
let header = block.header().clone();
if let Some(last_timestamp) = test_blocks.last_mut() {
if last_timestamp[0].timestamp == header.timestamp {
last_timestamp.push(header);
} else {
test_blocks.push(vec![header]);
}
} else {
test_blocks.push(vec![header]);
}
}
tracing::info!("blocks for testing: {test_blocks:#?}");

// Define invariants that every response should satisfy.
let check_invariants = |res: &TimeWindowQueryData, start, end, check_prev| {
let mut prev = res.prev.as_ref();
if let Some(prev) = prev {
if check_prev {
assert!(prev.timestamp < start);
}
} else {
// `prev` can only be `None` if the first block in the window is the genesis block.
assert_eq!(res.from().unwrap(), 0);
};
for header in &res.window {
assert!(start <= header.timestamp);
assert!(header.timestamp < end);
if let Some(prev) = prev {
assert!(prev.timestamp <= header.timestamp);
}
prev = Some(header);
}
if let Some(next) = &res.next {
assert!(next.timestamp >= end);
// If there is a `next`, there must be at least one previous block (either `prev`
// itself or the last block if the window is nonempty), so we can `unwrap` here.
assert!(next.timestamp >= prev.unwrap().timestamp);
}
};

let get_window = |start, end| {
let client = client.clone();
async move {
let res = client
.get(&format!("availability/headers/window/{start}/{end}"))
.send()
.await
.unwrap();
tracing::info!("window for timestamp range {start}-{end}: {res:#?}");
check_invariants(&res, start, end, true);
res
}
};

// Case 0: happy path. All blocks are available, including prev and next.
let start = test_blocks[1][0].timestamp;
let end = start + 1;
let res = get_window(start, end).await;
assert_eq!(res.prev.unwrap(), *test_blocks[0].last().unwrap());
assert_eq!(res.window, test_blocks[1]);
assert_eq!(res.next.unwrap(), test_blocks[2][0]);

// Case 1: no `prev`, start of window is before genesis.
let start = 0;
let end = test_blocks[0][0].timestamp + 1;
let res = get_window(start, end).await;
assert_eq!(res.prev, None);
assert_eq!(res.window, test_blocks[0]);
assert_eq!(res.next.unwrap(), test_blocks[1][0]);

// Case 2: no `next`, end of window is after the most recently sequenced block.
let start = test_blocks[2][0].timestamp;
let end = i64::MAX as u64;
let res = get_window(start, end).await;
assert_eq!(res.prev.unwrap(), *test_blocks[1].last().unwrap());
// There may have been more blocks sequenced since we grabbed `test_blocks`, so just check
// that the prefix of the window is correct.
assert_eq!(res.window[..test_blocks[2].len()], test_blocks[2]);
assert_eq!(res.next, None);
// Fetch more blocks using the `from` form of the endpoint. Start from the last block we had
// previously (ie fetch a slightly overlapping window) to ensure there is at least one block
// in the new window.
let from = test_blocks.iter().flatten().count() - 1;
let more: TimeWindowQueryData = client
.get(&format!("availability/headers/window/from/{from}/{end}",))
.send()
.await
.unwrap();
check_invariants(&more, start, end, false);
assert_eq!(
more.prev.as_ref().unwrap(),
test_blocks.iter().flatten().nth(from - 1).unwrap()
);
assert_eq!(
more.window[..res.window.len() - test_blocks[2].len() + 1],
res.window[test_blocks[2].len() - 1..]
);
assert_eq!(res.next, None);
// We should get the same result whether we query by block height or hash.
let more2: TimeWindowQueryData = client
.get(&format!(
"availability/headers/window/from/hash/{}/{}",
test_blocks[2].last().unwrap().commit(),
end
))
.send()
.await
.unwrap();
check_invariants(&more2, start, end, false);
assert_eq!(more2.from().unwrap(), more.from().unwrap());
assert_eq!(more2.prev, more.prev);
assert_eq!(more2.next, more.next);
assert_eq!(more2.window[..more.window.len()], more.window);

// Case 3: the window is empty.
let start = test_blocks[1][0].timestamp;
let end = start;
let res = get_window(start, end).await;
assert_eq!(res.prev.unwrap(), *test_blocks[0].last().unwrap());
assert_eq!(res.next.unwrap(), test_blocks[1][0]);
assert_eq!(res.window, vec![]);

// Case 5: no relevant blocks are available yet.
client
.get::<TimeWindowQueryData>(&format!(
"availability/headers/window/{}/{}",
i64::MAX - 1,
i64::MAX
))
.send()
.await
.unwrap_err();
}
}

#[cfg(test)]
Expand Down
12 changes: 1 addition & 11 deletions sequencer/src/api/data_source.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use super::{
endpoints::TimeWindowQueryData,
fs,
options::{Options, Query},
sql,
Expand All @@ -9,12 +8,11 @@ use async_std::sync::Arc;
use async_trait::async_trait;
use hotshot::types::SystemContextHandle;
use hotshot_query_service::{
availability::{AvailabilityDataSource, BlockId},
availability::AvailabilityDataSource,
data_source::{UpdateDataSource, VersionedDataSource},
fetching::provider::{AnyProvider, QueryServiceProvider},
node::NodeDataSource,
status::StatusDataSource,
QueryResult,
};
use hotshot_types::{data::ViewNumber, light_client::StateSignatureRequestBody};
use tide_disco::Url;
Expand Down Expand Up @@ -65,14 +63,6 @@ pub trait SequencerDataSource:
/// Any blocks in the data sources with number `from_block` or greater will be incorporated into
/// sequencer-specific data structures.
async fn refresh_indices(&mut self, from_block: usize) -> anyhow::Result<()>;

/// Retrieve a list of blocks whose timestamps fall within the window [start, end).
async fn window(&self, start: u64, end: u64) -> QueryResult<TimeWindowQueryData>;

/// Retrieve a list of blocks starting from `from` with timestamps less than `end`.
async fn window_from<ID>(&self, from: ID, end: u64) -> QueryResult<TimeWindowQueryData>
where
ID: Into<BlockId<SeqTypes>> + Send + Sync;
}

/// Provider for fetching missing data for the query service.
Expand Down
44 changes: 2 additions & 42 deletions sequencer/src/api/endpoints.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ use crate::{
block::payload::{parse_ns_payload, NamespaceProof},
network,
state::{BlockMerkleTree, FeeAccountProof, ValidatedState},
Header, NamespaceId, SeqTypes, Transaction,
NamespaceId, SeqTypes, Transaction,
};
use async_std::sync::{Arc, RwLock};
use commit::Committable;
use ethers::prelude::U256;
use futures::{try_join, FutureExt};
use hotshot_query_service::{
availability::{self, AvailabilityDataSource, BlockHash, CustomSnafu, FetchBlockSnafu},
availability::{self, AvailabilityDataSource, CustomSnafu, FetchBlockSnafu},
node, Error,
};
use hotshot_types::{data::ViewNumber, traits::node_implementation::ConsensusTime};
Expand All @@ -35,25 +35,6 @@ pub struct NamespaceProofQueryData {
pub transactions: Vec<Transaction>,
}

#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct TimeWindowQueryData {
pub window: Vec<Header>,
pub prev: Option<Header>,
pub next: Option<Header>,
}

impl TimeWindowQueryData {
/// The block height of the block that starts the window.
///
/// If the window is empty, this is the height of the block that ends the window.
pub fn from(&self) -> Option<u64> {
self.window
.first()
.or(self.next.as_ref())
.map(|header| header.height)
}
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct AccountQueryData {
pub balance: U256,
Expand Down Expand Up @@ -132,27 +113,6 @@ where
})
}
.boxed()
})?
.get("gettimestampwindow", |req, state| {
async move {
let end = req.integer_param("end")?;
let res = if let Some(height) = req.opt_integer_param("height")? {
state.inner().window_from::<usize>(height, end).await
} else if let Some(hash) = req.opt_blob_param("hash")? {
state
.inner()
.window_from::<BlockHash<SeqTypes>>(hash, end)
.await
} else {
let start: u64 = req.integer_param("start")?;
state.inner().window(start, end).await
};
res.map_err(|err| availability::Error::Custom {
message: err.to_string(),
status: err.status(),
})
}
.boxed()
})?;

Ok(api)
Expand Down
Loading

0 comments on commit 4b90848

Please sign in to comment.