From 4c11a4c8c6234158b5a866d3e43a099da9b25497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 14:00:58 +0100 Subject: [PATCH 01/12] Copy test utils into integration tests As a small side change it changes "supports_feature" to use zero-copy deserialization to &str. --- scylla/src/utils/test_utils.rs | 6 +- scylla/tests/integration/utils.rs | 110 ++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/scylla/src/utils/test_utils.rs b/scylla/src/utils/test_utils.rs index d15284d61e..0cdce2e0a0 100644 --- a/scylla/src/utils/test_utils.rs +++ b/scylla/src/utils/test_utils.rs @@ -44,15 +44,15 @@ pub(crate) async fn supports_feature(session: &Session, feature: &str) -> bool { return false; } - let (features,): (Option,) = session + let result = session .query_unpaged("SELECT supported_features FROM system.local", ()) .await .unwrap() .into_rows_result() - .unwrap() - .single_row() .unwrap(); + let (features,): (Option<&str>,) = result.single_row().unwrap(); + features .unwrap_or_default() .split(',') diff --git a/scylla/tests/integration/utils.rs b/scylla/tests/integration/utils.rs index 4d5b10f7a4..8afb4e9443 100644 --- a/scylla/tests/integration/utils.rs +++ b/scylla/tests/integration/utils.rs @@ -1,8 +1,14 @@ use futures::Future; +use scylla::frame::response::result::Row; +use scylla::transport::session_builder::{GenericSessionBuilder, SessionBuilderKind}; +use scylla::Session; use std::collections::HashMap; use std::env; use std::net::SocketAddr; +use std::num::NonZeroU32; use std::str::FromStr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use scylla_proxy::{Node, Proxy, ProxyError, RunningProxy, ShardAwareness}; @@ -14,6 +20,23 @@ pub(crate) fn setup_tracing() { .try_init(); } +static UNIQUE_COUNTER: AtomicUsize = AtomicUsize::new(0); + +#[allow(unused)] +pub(crate) fn unique_keyspace_name() -> String { + let cnt = UNIQUE_COUNTER.fetch_add(1, Ordering::SeqCst); + let name = format!( + "test_rust_{}_{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + cnt + ); + println!("Unique name: {}", name); + name +} + pub(crate) async fn test_with_3_node_cluster( shard_awareness: ShardAwareness, test: F, @@ -63,3 +86,90 @@ where running_proxy.finish().await } + +#[allow(unused)] +pub(crate) async fn supports_feature(session: &Session, feature: &str) -> bool { + // Cassandra doesn't have a concept of features, so first detect + // if there is the `supported_features` column in system.local + + let meta = session.get_cluster_data(); + let system_local = meta + .get_keyspace_info() + .get("system") + .unwrap() + .tables + .get("local") + .unwrap(); + + if !system_local.columns.contains_key("supported_features") { + return false; + } + + let result = session + .query_unpaged("SELECT supported_features FROM system.local", ()) + .await + .unwrap() + .into_rows_result() + .unwrap(); + + let (features,): (Option<&str>,) = result.single_row().unwrap(); + + features + .unwrap_or_default() + .split(',') + .any(|f| f == feature) +} + +#[allow(unused)] +pub(crate) async fn scylla_supports_tablets(session: &Session) -> bool { + let result = session + .query_unpaged( + "select column_name from system_schema.columns where + keyspace_name = 'system_schema' + and table_name = 'scylla_keyspaces' + and column_name = 'initial_tablets'", + &[], + ) + .await + .unwrap() + .into_rows_result(); + + result.is_ok_and(|rows_result| rows_result.single_row::().is_ok()) +} + +// Creates a generic session builder based on conditional compilation configuration +// For SessionBuilder of DefaultMode type, adds localhost to known hosts, as all of the tests +// connect to localhost. +#[allow(unused)] +pub(crate) fn create_new_session_builder() -> GenericSessionBuilder { + let session_builder = { + #[cfg(not(scylla_cloud_tests))] + { + use scylla::SessionBuilder; + + let uri = std::env::var("SCYLLA_URI").unwrap_or_else(|_| "127.0.0.1:9042".to_string()); + + SessionBuilder::new().known_node(uri) + } + + #[cfg(scylla_cloud_tests)] + { + use scylla::transport::session_builder::CloudMode; + use scylla::CloudSessionBuilder; + use std::path::Path; + + std::env::var("CLOUD_CONFIG_PATH") + .map(|config_path| CloudSessionBuilder::new(Path::new(&config_path))) + .expect("Failed to initialize CloudSessionBuilder") + .expect("CLOUD_CONFIG_PATH environment variable is missing") + } + }; + + // The reason why we enable so long waiting for TracingInfo is... Cassandra. (Yes, again.) + // In Cassandra Java Driver, the wait time for tracing info is 10 seconds, so here we do the same. + // However, as Scylla usually gets TracingInfo ready really fast (our default interval is hence 3ms), + // we stick to a not-so-much-terribly-long interval here. + session_builder + .tracing_info_fetch_attempts(NonZeroU32::new(200).unwrap()) + .tracing_info_fetch_interval(Duration::from_millis(50)) +} From 1503ff1832cdc2e3ba5a49aff112137ba822085c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 14:29:42 +0100 Subject: [PATCH 02/12] Integration tests: Move to new utils --- scylla/src/lib.rs | 6 ++---- scylla/src/utils/mod.rs | 4 +++- scylla/src/utils/test_utils.rs | 14 ++++---------- scylla/tests/integration/consistency.rs | 2 +- scylla/tests/integration/execution_profiles.rs | 3 +-- scylla/tests/integration/lwt_optimisation.rs | 6 +++--- scylla/tests/integration/retries.rs | 4 ++-- scylla/tests/integration/shards.rs | 7 ++++--- scylla/tests/integration/silent_prepare_query.rs | 4 ++-- .../integration/skip_metadata_optimization.rs | 4 ++-- scylla/tests/integration/tablets.rs | 9 +++------ 11 files changed, 27 insertions(+), 36 deletions(-) diff --git a/scylla/src/lib.rs b/scylla/src/lib.rs index 0bd45d7a14..f03edf7a9e 100644 --- a/scylla/src/lib.rs +++ b/scylla/src/lib.rs @@ -257,10 +257,8 @@ pub mod transport; pub(crate) mod utils; -/// This module is NOT part of the public API (it is `pub` only for internal use of integration tests). -/// Future minor releases are free to introduce breaking API changes inside it. -#[doc(hidden)] -pub use utils::test_utils; +#[cfg(test)] +pub(crate) use utils::test_utils; pub use statement::batch; pub use statement::prepared_statement; diff --git a/scylla/src/utils/mod.rs b/scylla/src/utils/mod.rs index c83adbf7e7..bf0f94e752 100644 --- a/scylla/src/utils/mod.rs +++ b/scylla/src/utils/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod parse; pub(crate) mod pretty; -pub mod test_utils; + +#[cfg(test)] +pub(crate) mod test_utils; diff --git a/scylla/src/utils/test_utils.rs b/scylla/src/utils/test_utils.rs index 0cdce2e0a0..68035fcee3 100644 --- a/scylla/src/utils/test_utils.rs +++ b/scylla/src/utils/test_utils.rs @@ -1,9 +1,6 @@ -use scylla_cql::frame::response::result::Row; - -#[cfg(test)] use crate::transport::session_builder::{GenericSessionBuilder, SessionBuilderKind}; use crate::Session; -#[cfg(test)] +use scylla_cql::frame::response::result::Row; use std::{num::NonZeroU32, time::Duration}; use std::{ sync::atomic::{AtomicUsize, Ordering}, @@ -12,7 +9,7 @@ use std::{ static UNIQUE_COUNTER: AtomicUsize = AtomicUsize::new(0); -pub fn unique_keyspace_name() -> String { +pub(crate) fn unique_keyspace_name() -> String { let cnt = UNIQUE_COUNTER.fetch_add(1, Ordering::SeqCst); let name = format!( "test_rust_{}_{}", @@ -26,7 +23,6 @@ pub fn unique_keyspace_name() -> String { name } -#[cfg(test)] pub(crate) async fn supports_feature(session: &Session, feature: &str) -> bool { // Cassandra doesn't have a concept of features, so first detect // if there is the `supported_features` column in system.local @@ -62,8 +58,7 @@ pub(crate) async fn supports_feature(session: &Session, feature: &str) -> bool { // Creates a generic session builder based on conditional compilation configuration // For SessionBuilder of DefaultMode type, adds localhost to known hosts, as all of the tests // connect to localhost. -#[cfg(test)] -pub fn create_new_session_builder() -> GenericSessionBuilder { +pub(crate) fn create_new_session_builder() -> GenericSessionBuilder { let session_builder = { #[cfg(not(scylla_cloud_tests))] { @@ -96,7 +91,7 @@ pub fn create_new_session_builder() -> GenericSessionBuilder bool { +pub(crate) async fn scylla_supports_tablets(session: &Session) -> bool { let result = session .query_unpaged( "select column_name from system_schema.columns where @@ -112,7 +107,6 @@ pub async fn scylla_supports_tablets(session: &Session) -> bool { result.is_ok_and(|rows_result| rows_result.single_row::().is_ok()) } -#[cfg(test)] pub(crate) fn setup_tracing() { let _ = tracing_subscriber::fmt::fmt() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) diff --git a/scylla/tests/integration/consistency.rs b/scylla/tests/integration/consistency.rs index 09780066ac..aee410f762 100644 --- a/scylla/tests/integration/consistency.rs +++ b/scylla/tests/integration/consistency.rs @@ -1,11 +1,11 @@ use crate::utils::{setup_tracing, test_with_3_node_cluster}; +use crate::utils::unique_keyspace_name; use scylla::execution_profile::{ExecutionProfileBuilder, ExecutionProfileHandle}; use scylla::load_balancing::{DefaultPolicy, LoadBalancingPolicy, RoutingInfo}; use scylla::prepared_statement::PreparedStatement; use scylla::retry_policy::FallthroughRetryPolicy; use scylla::routing::{Shard, Token}; -use scylla::test_utils::unique_keyspace_name; use scylla::transport::NodeRef; use scylla::Session; use scylla_cql::frame::response::result::TableSpec; diff --git a/scylla/tests/integration/execution_profiles.rs b/scylla/tests/integration/execution_profiles.rs index 0a49bae785..cdfde1c32c 100644 --- a/scylla/tests/integration/execution_profiles.rs +++ b/scylla/tests/integration/execution_profiles.rs @@ -1,7 +1,7 @@ use std::ops::Deref; use std::sync::Arc; -use crate::utils::{setup_tracing, test_with_3_node_cluster}; +use crate::utils::{setup_tracing, test_with_3_node_cluster, unique_keyspace_name}; use assert_matches::assert_matches; use scylla::batch::BatchStatement; use scylla::batch::{Batch, BatchType}; @@ -13,7 +13,6 @@ use scylla::{ load_balancing::{LoadBalancingPolicy, RoutingInfo}, retry_policy::{RetryPolicy, RetrySession}, speculative_execution::SpeculativeExecutionPolicy, - test_utils::unique_keyspace_name, transport::ClusterData, ExecutionProfile, SessionBuilder, }; diff --git a/scylla/tests/integration/lwt_optimisation.rs b/scylla/tests/integration/lwt_optimisation.rs index ca56cff930..9fd6f073f0 100644 --- a/scylla/tests/integration/lwt_optimisation.rs +++ b/scylla/tests/integration/lwt_optimisation.rs @@ -1,7 +1,7 @@ -use crate::utils::{setup_tracing, test_with_3_node_cluster}; +use crate::utils::{ + scylla_supports_tablets, setup_tracing, test_with_3_node_cluster, unique_keyspace_name, +}; use scylla::retry_policy::FallthroughRetryPolicy; -use scylla::test_utils::scylla_supports_tablets; -use scylla::test_utils::unique_keyspace_name; use scylla::transport::session::Session; use scylla::{ExecutionProfile, SessionBuilder}; use scylla_cql::frame::protocol_features::ProtocolFeatures; diff --git a/scylla/tests/integration/retries.rs b/scylla/tests/integration/retries.rs index 43cbf58074..a21de6d619 100644 --- a/scylla/tests/integration/retries.rs +++ b/scylla/tests/integration/retries.rs @@ -1,10 +1,10 @@ -use crate::utils::{setup_tracing, test_with_3_node_cluster}; +use crate::utils::{setup_tracing, test_with_3_node_cluster, unique_keyspace_name}; +use scylla::query::Query; use scylla::retry_policy::FallthroughRetryPolicy; use scylla::speculative_execution::SimpleSpeculativeExecutionPolicy; use scylla::transport::session::Session; use scylla::ExecutionProfile; use scylla::SessionBuilder; -use scylla::{query::Query, test_utils::unique_keyspace_name}; use std::sync::Arc; use std::time::Duration; use tracing::info; diff --git a/scylla/tests/integration/shards.rs b/scylla/tests/integration/shards.rs index b22cfc397b..17f3c1ceb0 100644 --- a/scylla/tests/integration/shards.rs +++ b/scylla/tests/integration/shards.rs @@ -1,8 +1,9 @@ use std::sync::Arc; -use crate::utils::{setup_tracing, test_with_3_node_cluster}; -use scylla::test_utils::scylla_supports_tablets; -use scylla::{test_utils::unique_keyspace_name, SessionBuilder}; +use crate::utils::{ + scylla_supports_tablets, setup_tracing, test_with_3_node_cluster, unique_keyspace_name, +}; +use scylla::SessionBuilder; use tokio::sync::mpsc; use scylla_proxy::TargetShard; diff --git a/scylla/tests/integration/silent_prepare_query.rs b/scylla/tests/integration/silent_prepare_query.rs index 93950206a5..ed259914ef 100644 --- a/scylla/tests/integration/silent_prepare_query.rs +++ b/scylla/tests/integration/silent_prepare_query.rs @@ -1,7 +1,7 @@ -use crate::utils::{setup_tracing, test_with_3_node_cluster}; +use crate::utils::{setup_tracing, test_with_3_node_cluster, unique_keyspace_name}; +use scylla::query::Query; use scylla::Session; use scylla::SessionBuilder; -use scylla::{query::Query, test_utils::unique_keyspace_name}; use scylla_proxy::{ Condition, ProxyError, Reaction, RequestOpcode, RequestReaction, RequestRule, ShardAwareness, WorkerError, diff --git a/scylla/tests/integration/skip_metadata_optimization.rs b/scylla/tests/integration/skip_metadata_optimization.rs index dba646e895..1260eda326 100644 --- a/scylla/tests/integration/skip_metadata_optimization.rs +++ b/scylla/tests/integration/skip_metadata_optimization.rs @@ -1,5 +1,5 @@ -use crate::utils::{setup_tracing, test_with_3_node_cluster}; -use scylla::{prepared_statement::PreparedStatement, test_utils::unique_keyspace_name}; +use crate::utils::{setup_tracing, test_with_3_node_cluster, unique_keyspace_name}; +use scylla::prepared_statement::PreparedStatement; use scylla::{Session, SessionBuilder}; use scylla_cql::frame::request::query::{PagingState, PagingStateResponse}; use scylla_cql::frame::types; diff --git a/scylla/tests/integration/tablets.rs b/scylla/tests/integration/tablets.rs index 9dbb5d31ab..446ac170e1 100644 --- a/scylla/tests/integration/tablets.rs +++ b/scylla/tests/integration/tablets.rs @@ -1,7 +1,9 @@ use std::sync::Arc; +use crate::utils::scylla_supports_tablets; use crate::utils::setup_tracing; use crate::utils::test_with_3_node_cluster; +use crate::utils::unique_keyspace_name; use futures::future::try_join_all; use futures::TryStreamExt; @@ -12,7 +14,6 @@ use scylla::load_balancing::RoutingInfo; use scylla::prepared_statement::PreparedStatement; use scylla::query::Query; use scylla::serialize::row::SerializeRow; -use scylla::test_utils::unique_keyspace_name; use scylla::transport::ClusterData; use scylla::transport::Node; use scylla::transport::NodeRef; @@ -300,7 +301,7 @@ async fn test_default_policy_is_tablet_aware() { .await .unwrap(); - if !scylla::test_utils::scylla_supports_tablets(&session).await { + if !scylla_supports_tablets(&session).await { tracing::warn!("Skipping test because this Scylla version doesn't support tablets"); return running_proxy; } @@ -418,8 +419,6 @@ async fn test_default_policy_is_tablet_aware() { #[tokio::test] #[ntest::timeout(30000)] async fn test_tablet_feedback_not_sent_for_unprepared_queries() { - use scylla::test_utils::scylla_supports_tablets; - setup_tracing(); const TABLET_COUNT: usize = 16; @@ -492,8 +491,6 @@ async fn test_tablet_feedback_not_sent_for_unprepared_queries() { #[ntest::timeout(30000)] #[ignore] async fn test_lwt_optimization_works_with_tablets() { - use scylla::test_utils::scylla_supports_tablets; - setup_tracing(); const TABLET_COUNT: usize = 16; From aab0e8b176b6a1d0ba9dd7a14022e4c60d375dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 15:25:34 +0100 Subject: [PATCH 03/12] cql_collections_test: Move to integration tests --- scylla/src/transport/mod.rs | 2 -- .../integration/cql_collections.rs} | 11 +++++------ scylla/tests/integration/main.rs | 1 + scylla/tests/integration/utils.rs | 12 ++++++++++++ 4 files changed, 18 insertions(+), 8 deletions(-) rename scylla/{src/transport/cql_collections_test.rs => tests/integration/cql_collections.rs} (96%) diff --git a/scylla/src/transport/mod.rs b/scylla/src/transport/mod.rs index be4cfa37ba..f46c18b410 100644 --- a/scylla/src/transport/mod.rs +++ b/scylla/src/transport/mod.rs @@ -28,8 +28,6 @@ pub use scylla_cql::frame::request::query::{PagingState, PagingStateResponse}; #[cfg(test)] mod authenticate_test; #[cfg(test)] -mod cql_collections_test; -#[cfg(test)] mod session_test; #[cfg(test)] mod silent_prepare_batch_test; diff --git a/scylla/src/transport/cql_collections_test.rs b/scylla/tests/integration/cql_collections.rs similarity index 96% rename from scylla/src/transport/cql_collections_test.rs rename to scylla/tests/integration/cql_collections.rs index 9cdb34ce5a..c8e783547f 100644 --- a/scylla/src/transport/cql_collections_test.rs +++ b/scylla/tests/integration/cql_collections.rs @@ -1,9 +1,8 @@ -use crate::deserialize::DeserializeOwnedValue; -use crate::transport::session::Session; - -use crate::frame::response::result::CqlValue; -use crate::test_utils::{create_new_session_builder, setup_tracing}; -use crate::utils::test_utils::unique_keyspace_name; +use crate::utils::{ + create_new_session_builder, setup_tracing, unique_keyspace_name, DeserializeOwnedValue, +}; +use scylla::frame::response::result::CqlValue; +use scylla::Session; use scylla_cql::types::serialize::value::SerializeValue; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index ef190f1237..f62ca47dfb 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -1,4 +1,5 @@ mod consistency; +mod cql_collections; mod execution_profiles; mod hygiene; mod lwt_optimisation; diff --git a/scylla/tests/integration/utils.rs b/scylla/tests/integration/utils.rs index 8afb4e9443..4ec8539c5c 100644 --- a/scylla/tests/integration/utils.rs +++ b/scylla/tests/integration/utils.rs @@ -1,4 +1,5 @@ use futures::Future; +use scylla::deserialize::DeserializeValue; use scylla::frame::response::result::Row; use scylla::transport::session_builder::{GenericSessionBuilder, SessionBuilderKind}; use scylla::Session; @@ -173,3 +174,14 @@ pub(crate) fn create_new_session_builder() -> GenericSessionBuilder DeserializeValue<'frame, 'metadata> +{ +} +impl DeserializeOwnedValue for T where + T: for<'frame, 'metadata> DeserializeValue<'frame, 'metadata> +{ +} From 3d50042324eb831dc077e34f2517b899df3de11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 14:41:25 +0100 Subject: [PATCH 04/12] atuhenticate_test: Move to integration --- scylla/src/transport/mod.rs | 2 -- .../integration/authenticate.rs} | 9 ++++----- scylla/tests/integration/main.rs | 1 + 3 files changed, 5 insertions(+), 7 deletions(-) rename scylla/{src/transport/authenticate_test.rs => tests/integration/authenticate.rs} (90%) diff --git a/scylla/src/transport/mod.rs b/scylla/src/transport/mod.rs index f46c18b410..a0a5a5ad93 100644 --- a/scylla/src/transport/mod.rs +++ b/scylla/src/transport/mod.rs @@ -25,8 +25,6 @@ pub use connection::SelfIdentity; pub use execution_profile::ExecutionProfile; pub use scylla_cql::frame::request::query::{PagingState, PagingStateResponse}; -#[cfg(test)] -mod authenticate_test; #[cfg(test)] mod session_test; #[cfg(test)] diff --git a/scylla/src/transport/authenticate_test.rs b/scylla/tests/integration/authenticate.rs similarity index 90% rename from scylla/src/transport/authenticate_test.rs rename to scylla/tests/integration/authenticate.rs index 78e72dea40..babdb0d5c8 100644 --- a/scylla/src/transport/authenticate_test.rs +++ b/scylla/tests/integration/authenticate.rs @@ -1,8 +1,7 @@ -use crate::authentication::{AuthError, AuthenticatorProvider, AuthenticatorSession}; -use crate::test_utils::setup_tracing; -use crate::utils::test_utils::unique_keyspace_name; +use crate::utils::{setup_tracing, unique_keyspace_name}; use async_trait::async_trait; use bytes::{BufMut, BytesMut}; +use scylla::authentication::{AuthError, AuthenticatorProvider, AuthenticatorSession}; use std::sync::Arc; #[tokio::test] @@ -13,7 +12,7 @@ async fn authenticate_superuser() { println!("Connecting to {} with cassandra superuser ...", uri); - let session = crate::SessionBuilder::new() + let session = scylla::SessionBuilder::new() .known_node(uri) .user("cassandra", "cassandra") .build() @@ -72,7 +71,7 @@ async fn custom_authentication() { println!("Connecting to {} with cassandra superuser ...", uri); - let session = crate::SessionBuilder::new() + let session = scylla::SessionBuilder::new() .known_node(uri) .authenticator_provider(Arc::new(CustomAuthenticatorProvider)) .build() diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index f62ca47dfb..67371533a7 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -1,3 +1,4 @@ +mod authenticate; mod consistency; mod cql_collections; mod execution_profiles; From d45c5c0ff8317902dfb07d0a3ec230279b6748ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 14:48:49 +0100 Subject: [PATCH 05/12] cql_types_test: Move to integration --- scylla/src/transport/mod.rs | 2 -- .../integration/cql_types.rs} | 26 ++++++++----------- scylla/tests/integration/main.rs | 1 + 3 files changed, 12 insertions(+), 17 deletions(-) rename scylla/{src/transport/cql_types_test.rs => tests/integration/cql_types.rs} (98%) diff --git a/scylla/src/transport/mod.rs b/scylla/src/transport/mod.rs index a0a5a5ad93..d41b810175 100644 --- a/scylla/src/transport/mod.rs +++ b/scylla/src/transport/mod.rs @@ -30,8 +30,6 @@ mod session_test; #[cfg(test)] mod silent_prepare_batch_test; -#[cfg(test)] -mod cql_types_test; #[cfg(test)] mod cql_value_test; #[cfg(test)] diff --git a/scylla/src/transport/cql_types_test.rs b/scylla/tests/integration/cql_types.rs similarity index 98% rename from scylla/src/transport/cql_types_test.rs rename to scylla/tests/integration/cql_types.rs index 30f406a1db..62c1b8f5ff 100644 --- a/scylla/src/transport/cql_types_test.rs +++ b/scylla/tests/integration/cql_types.rs @@ -1,19 +1,19 @@ -use crate as scylla; -use crate::deserialize::DeserializeOwnedValue; -use crate::frame::response::result::CqlValue; -use crate::frame::value::{Counter, CqlDate, CqlTime, CqlTimestamp}; -use crate::test_utils::{create_new_session_builder, scylla_supports_tablets, setup_tracing}; -use crate::transport::session::Session; -use crate::utils::test_utils::unique_keyspace_name; use itertools::Itertools; -use scylla_cql::frame::value::{CqlTimeuuid, CqlVarint}; -use scylla_cql::types::serialize::value::SerializeValue; -use scylla_macros::{DeserializeValue, SerializeValue}; +use scylla::frame::response::result::CqlValue; +use scylla::frame::value::{Counter, CqlDate, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint}; +use scylla::serialize::value::SerializeValue; +use scylla::Session; +use scylla::{DeserializeValue, SerializeValue}; use std::cmp::PartialEq; use std::fmt::Debug; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::str::FromStr; +use crate::utils::{ + create_new_session_builder, scylla_supports_tablets, setup_tracing, unique_keyspace_name, + DeserializeOwnedValue, +}; + // Used to prepare a table for test // Creates a new keyspace, without tablets if requested and the ScyllaDB instance supports them. // Drops and creates table {table_name} (id int PRIMARY KEY, val {type_name}) @@ -1572,7 +1572,6 @@ async fn test_udt_after_schema_update() { .unwrap(); #[derive(SerializeValue, DeserializeValue, Debug, PartialEq)] - #[scylla(crate = crate)] struct UdtV1 { first: i32, second: bool, @@ -1796,7 +1795,6 @@ async fn test_udt_with_missing_field() { } #[derive(SerializeValue)] - #[scylla(crate = crate)] struct UdtV1 { first: i32, second: bool, @@ -1822,7 +1820,6 @@ async fn test_udt_with_missing_field() { id += 1; #[derive(SerializeValue)] - #[scylla(crate = crate)] struct UdtV2 { first: i32, second: bool, @@ -1850,7 +1847,6 @@ async fn test_udt_with_missing_field() { id += 1; #[derive(SerializeValue)] - #[scylla(crate = crate)] struct UdtV3 { first: i32, second: bool, @@ -1878,7 +1874,7 @@ async fn test_udt_with_missing_field() { id += 1; #[derive(SerializeValue)] - #[scylla(crate = crate, flavor="enforce_order")] + #[scylla(flavor = "enforce_order")] struct UdtV4 { first: i32, second: bool, diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index 67371533a7..83daead04b 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -1,6 +1,7 @@ mod authenticate; mod consistency; mod cql_collections; +mod cql_types; mod execution_profiles; mod hygiene; mod lwt_optimisation; From 456df3382c971c53320f1960997fe922dae97685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 14:53:18 +0100 Subject: [PATCH 06/12] cql_value_test: Move to integration --- scylla/src/transport/mod.rs | 2 -- .../cql_value_test.rs => tests/integration/cql_value.rs} | 9 ++++----- scylla/tests/integration/main.rs | 1 + 3 files changed, 5 insertions(+), 7 deletions(-) rename scylla/{src/transport/cql_value_test.rs => tests/integration/cql_value.rs} (95%) diff --git a/scylla/src/transport/mod.rs b/scylla/src/transport/mod.rs index d41b810175..f84e068d02 100644 --- a/scylla/src/transport/mod.rs +++ b/scylla/src/transport/mod.rs @@ -30,8 +30,6 @@ mod session_test; #[cfg(test)] mod silent_prepare_batch_test; -#[cfg(test)] -mod cql_value_test; #[cfg(test)] mod large_batch_statements_test; diff --git a/scylla/src/transport/cql_value_test.rs b/scylla/tests/integration/cql_value.rs similarity index 95% rename from scylla/src/transport/cql_value_test.rs rename to scylla/tests/integration/cql_value.rs index 932b72934b..c0e55562d8 100644 --- a/scylla/src/transport/cql_value_test.rs +++ b/scylla/tests/integration/cql_value.rs @@ -1,11 +1,10 @@ use assert_matches::assert_matches; -use crate::frame::response::result::CqlValue; -use crate::frame::value::CqlDuration; +use scylla::frame::response::result::CqlValue; +use scylla::frame::value::CqlDuration; +use scylla::Session; -use crate::test_utils::{create_new_session_builder, setup_tracing}; -use crate::utils::test_utils::unique_keyspace_name; -use crate::Session; +use crate::utils::{create_new_session_builder, setup_tracing, unique_keyspace_name}; #[tokio::test] async fn test_cqlvalue_udt() { diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index 83daead04b..d6a8490318 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -2,6 +2,7 @@ mod authenticate; mod consistency; mod cql_collections; mod cql_types; +mod cql_value; mod execution_profiles; mod hygiene; mod lwt_optimisation; From 8297a01149d226037c9c08045712d5ab9266240e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 14:57:39 +0100 Subject: [PATCH 07/12] large_batch_statements_test: Move to integration --- scylla/src/transport/mod.rs | 3 --- .../integration/large_batch_statements.rs} | 15 ++++++--------- scylla/tests/integration/main.rs | 1 + 3 files changed, 7 insertions(+), 12 deletions(-) rename scylla/{src/transport/large_batch_statements_test.rs => tests/integration/large_batch_statements.rs} (88%) diff --git a/scylla/src/transport/mod.rs b/scylla/src/transport/mod.rs index f84e068d02..7730a101e7 100644 --- a/scylla/src/transport/mod.rs +++ b/scylla/src/transport/mod.rs @@ -30,8 +30,5 @@ mod session_test; #[cfg(test)] mod silent_prepare_batch_test; -#[cfg(test)] -mod large_batch_statements_test; - pub use cluster::ClusterData; pub use node::{KnownNode, Node, NodeAddr, NodeRef}; diff --git a/scylla/src/transport/large_batch_statements_test.rs b/scylla/tests/integration/large_batch_statements.rs similarity index 88% rename from scylla/src/transport/large_batch_statements_test.rs rename to scylla/tests/integration/large_batch_statements.rs index 7e8fc482c3..3d6757165b 100644 --- a/scylla/src/transport/large_batch_statements_test.rs +++ b/scylla/tests/integration/large_batch_statements.rs @@ -1,14 +1,11 @@ use assert_matches::assert_matches; -use crate::batch::BatchType; -use crate::query::Query; -use crate::test_utils::setup_tracing; -use crate::transport::errors::{BadQuery, QueryError}; -use crate::{ - batch::Batch, - test_utils::{create_new_session_builder, unique_keyspace_name}, - QueryResult, Session, -}; +use crate::utils::{create_new_session_builder, setup_tracing, unique_keyspace_name}; +use scylla::batch::Batch; +use scylla::batch::BatchType; +use scylla::query::Query; +use scylla::transport::errors::{BadQuery, QueryError}; +use scylla::{QueryResult, Session}; #[tokio::test] async fn test_large_batch_statements() { diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index d6a8490318..bfe21f0c1e 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -5,6 +5,7 @@ mod cql_types; mod cql_value; mod execution_profiles; mod hygiene; +mod large_batch_statements; mod lwt_optimisation; mod new_session; mod retries; From 61e2a8653cc695006afad8472ee74e2d8394fb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 15:00:56 +0100 Subject: [PATCH 08/12] silent_prepare_batch_test: Move to integration --- scylla/src/transport/mod.rs | 2 -- scylla/tests/integration/main.rs | 1 + .../integration/silent_prepare_batch.rs} | 11 +++++------ 3 files changed, 6 insertions(+), 8 deletions(-) rename scylla/{src/transport/silent_prepare_batch_test.rs => tests/integration/silent_prepare_batch.rs} (95%) diff --git a/scylla/src/transport/mod.rs b/scylla/src/transport/mod.rs index 7730a101e7..55184aadc6 100644 --- a/scylla/src/transport/mod.rs +++ b/scylla/src/transport/mod.rs @@ -27,8 +27,6 @@ pub use scylla_cql::frame::request::query::{PagingState, PagingStateResponse}; #[cfg(test)] mod session_test; -#[cfg(test)] -mod silent_prepare_batch_test; pub use cluster::ClusterData; pub use node::{KnownNode, Node, NodeAddr, NodeRef}; diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index bfe21f0c1e..8a9d22db02 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -11,6 +11,7 @@ mod new_session; mod retries; mod self_identity; mod shards; +mod silent_prepare_batch; mod silent_prepare_query; mod skip_metadata_optimization; mod tablets; diff --git a/scylla/src/transport/silent_prepare_batch_test.rs b/scylla/tests/integration/silent_prepare_batch.rs similarity index 95% rename from scylla/src/transport/silent_prepare_batch_test.rs rename to scylla/tests/integration/silent_prepare_batch.rs index bca8ef183a..3f86ae09f1 100644 --- a/scylla/src/transport/silent_prepare_batch_test.rs +++ b/scylla/tests/integration/silent_prepare_batch.rs @@ -1,9 +1,8 @@ -use crate::{ - batch::Batch, - prepared_statement::PreparedStatement, - test_utils::{create_new_session_builder, setup_tracing, unique_keyspace_name}, - Session, -}; +use crate::utils::{create_new_session_builder, setup_tracing, unique_keyspace_name}; +use scylla::batch::Batch; +use scylla::prepared_statement::PreparedStatement; +use scylla::Session; + use std::collections::BTreeSet; #[tokio::test] From d8a06b79d776e986fcc46a4ffa2e26d5e37587be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 15:18:31 +0100 Subject: [PATCH 09/12] history.rs: Move some tests to integration --- scylla/src/history.rs | 249 +---------------------- scylla/tests/integration/history.rs | 293 ++++++++++++++++++++++++++++ scylla/tests/integration/main.rs | 1 + 3 files changed, 296 insertions(+), 247 deletions(-) create mode 100644 scylla/tests/integration/history.rs diff --git a/scylla/src/history.rs b/scylla/src/history.rs index a055f91a39..b80dcf15ec 100644 --- a/scylla/src/history.rs +++ b/scylla/src/history.rs @@ -449,28 +449,21 @@ fn write_fiber_attempts(fiber: &FiberHistory, f: &mut std::fmt::Formatter<'_>) - #[cfg(test)] mod tests { - use std::{ - net::{IpAddr, Ipv4Addr, SocketAddr}, - sync::Arc, - }; + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use crate::{ - query::Query, retry_policy::RetryDecision, test_utils::setup_tracing, transport::errors::{DbError, QueryError}, - utils::test_utils::unique_keyspace_name, }; use super::{ AttemptId, AttemptResult, HistoryCollector, HistoryListener, QueryHistoryResult, QueryId, SpeculativeId, StructuredHistory, TimePoint, }; - use crate::test_utils::create_new_session_builder; use assert_matches::assert_matches; use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; - use futures::StreamExt as _; - use scylla_cql::{frame::response::result::Row, Consistency}; + use scylla_cql::Consistency; // Set a single time for all timestamps within StructuredHistory. // HistoryCollector sets the timestamp to current time which changes with each test. @@ -510,53 +503,6 @@ mod tests { history } - // Set a single node for all attempts within StructuredHistory. - // When running against real life nodes this address may change, - // setting it to one value makes it possible to run tests consistently. - fn set_one_node(mut history: StructuredHistory) -> StructuredHistory { - let the_node: SocketAddr = node1_addr(); - - for query in &mut history.queries { - for fiber in std::iter::once(&mut query.non_speculative_fiber) - .chain(query.speculative_fibers.iter_mut()) - { - for attempt in &mut fiber.attempts { - attempt.node_addr = the_node; - } - } - } - - history - } - - // Set a single error message for all DbErrors within StructuredHistory. - // The error message changes between Scylla/Cassandra/their versions. - // Setting it to one value makes it possible to run tests consistently. - fn set_one_db_error_message(mut history: StructuredHistory) -> StructuredHistory { - let set_msg = |err: &mut QueryError| { - if let QueryError::DbError(_, msg) = err { - *msg = "Error message from database".to_string(); - } - }; - - for query in &mut history.queries { - if let Some(QueryHistoryResult::Error(_, err)) = &mut query.result { - set_msg(err); - } - for fiber in std::iter::once(&mut query.non_speculative_fiber) - .chain(query.speculative_fibers.iter_mut()) - { - for attempt in &mut fiber.attempts { - if let Some(AttemptResult::Error(_, err, _)) = &mut attempt.result { - set_msg(err); - } - } - } - } - - history - } - fn node1_addr() -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 19042) } @@ -913,195 +859,4 @@ mod tests { "; assert_eq!(displayed, format!("{}", set_one_time(history))); } - - #[tokio::test] - async fn successful_query_history() { - setup_tracing(); - let session = create_new_session_builder().build().await.unwrap(); - - let mut query = Query::new("SELECT * FROM system.local"); - let history_collector = Arc::new(HistoryCollector::new()); - query.set_history_listener(history_collector.clone()); - - session.query_unpaged(query.clone(), ()).await.unwrap(); - - let history: StructuredHistory = history_collector.clone_structured_history(); - - let displayed = "Queries History: -=== Query #0 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Success at 2022-02-22 20:22:22 UTC -| -| Query successful at 2022-02-22 20:22:22 UTC -================= -"; - assert_eq!( - displayed, - format!( - "{}", - set_one_db_error_message(set_one_node(set_one_time(history))) - ) - ); - - // Prepared queries retain the history listener set in Query. - let prepared = session.prepare(query).await.unwrap(); - session.execute_unpaged(&prepared, ()).await.unwrap(); - - let history2: StructuredHistory = history_collector.clone_structured_history(); - - let displayed2 = "Queries History: -=== Query #0 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Success at 2022-02-22 20:22:22 UTC -| -| Query successful at 2022-02-22 20:22:22 UTC -================= -=== Query #1 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Success at 2022-02-22 20:22:22 UTC -| -| Query successful at 2022-02-22 20:22:22 UTC -================= -"; - assert_eq!( - displayed2, - format!( - "{}", - set_one_db_error_message(set_one_node(set_one_time(history2))) - ) - ); - } - - #[tokio::test] - async fn failed_query_history() { - setup_tracing(); - let session = create_new_session_builder().build().await.unwrap(); - - let mut query = Query::new("This isnt even CQL"); - let history_collector = Arc::new(HistoryCollector::new()); - query.set_history_listener(history_collector.clone()); - - assert!(session.query_unpaged(query.clone(), ()).await.is_err()); - - let history: StructuredHistory = history_collector.clone_structured_history(); - - let displayed = -"Queries History: -=== Query #0 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Error at 2022-02-22 20:22:22 UTC -| Error: Database returned an error: The submitted query has a syntax error, Error message: Error message from database -| Retry decision: DontRetry -| -| Query failed at 2022-02-22 20:22:22 UTC -| Error: Database returned an error: The submitted query has a syntax error, Error message: Error message from database -================= -"; - assert_eq!( - displayed, - format!( - "{}", - set_one_db_error_message(set_one_node(set_one_time(history))) - ) - ); - } - - #[tokio::test] - async fn iterator_query_history() { - setup_tracing(); - let session = create_new_session_builder().build().await.unwrap(); - let ks = unique_keyspace_name(); - session - .query_unpaged(format!("CREATE KEYSPACE {} WITH REPLICATION = {{'class' : 'NetworkTopologyStrategy', 'replication_factor' : 1}}", ks), &[]) - .await - .unwrap(); - session.use_keyspace(ks, true).await.unwrap(); - - session - .query_unpaged("CREATE TABLE t (p int primary key)", ()) - .await - .unwrap(); - for i in 0..32 { - session - .query_unpaged("INSERT INTO t (p) VALUES (?)", (i,)) - .await - .unwrap(); - } - - let mut iter_query: Query = Query::new("SELECT * FROM t"); - iter_query.set_page_size(8); - let history_collector = Arc::new(HistoryCollector::new()); - iter_query.set_history_listener(history_collector.clone()); - - let mut rows_iterator = session - .query_iter(iter_query, ()) - .await - .unwrap() - .rows_stream::() - .unwrap(); - while let Some(_row) = rows_iterator.next().await { - // Receive rows... - } - - let history = history_collector.clone_structured_history(); - - assert!(history.queries.len() >= 4); - - let displayed_prefix = "Queries History: -=== Query #0 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Success at 2022-02-22 20:22:22 UTC -| -| Query successful at 2022-02-22 20:22:22 UTC -================= -=== Query #1 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Success at 2022-02-22 20:22:22 UTC -| -| Query successful at 2022-02-22 20:22:22 UTC -================= -=== Query #2 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Success at 2022-02-22 20:22:22 UTC -| -| Query successful at 2022-02-22 20:22:22 UTC -================= -=== Query #3 === -| start_time: 2022-02-22 20:22:22 UTC -| Non-speculative attempts: -| - Attempt #0 sent to 127.0.0.1:19042 -| request send time: 2022-02-22 20:22:22 UTC -| Success at 2022-02-22 20:22:22 UTC -| -| Query successful at 2022-02-22 20:22:22 UTC -================= -"; - let displayed_str = format!( - "{}", - set_one_db_error_message(set_one_node(set_one_time(history))) - ); - - assert!(displayed_str.starts_with(displayed_prefix),); - } } diff --git a/scylla/tests/integration/history.rs b/scylla/tests/integration/history.rs new file mode 100644 index 0000000000..1d4f89df8d --- /dev/null +++ b/scylla/tests/integration/history.rs @@ -0,0 +1,293 @@ +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::Arc; + +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use futures::StreamExt; +use scylla::frame::response::result::Row; +use scylla::history::{ + AttemptResult, HistoryCollector, QueryHistoryResult, StructuredHistory, TimePoint, +}; +use scylla::query::Query; +use scylla::transport::errors::QueryError; + +use crate::utils::{create_new_session_builder, setup_tracing, unique_keyspace_name}; + +// Set a single time for all timestamps within StructuredHistory. +// HistoryCollector sets the timestamp to current time which changes with each test. +// Setting it to one makes it possible to test displaying consistently. +fn set_one_time(mut history: StructuredHistory) -> StructuredHistory { + let the_time: TimePoint = DateTime::::from_naive_utc_and_offset( + NaiveDateTime::new( + NaiveDate::from_ymd_opt(2022, 2, 22).unwrap(), + NaiveTime::from_hms_opt(20, 22, 22).unwrap(), + ), + Utc, + ); + + for query in &mut history.queries { + query.start_time = the_time; + match &mut query.result { + Some(QueryHistoryResult::Success(succ_time)) => *succ_time = the_time, + Some(QueryHistoryResult::Error(err_time, _)) => *err_time = the_time, + None => {} + }; + + for fiber in std::iter::once(&mut query.non_speculative_fiber) + .chain(query.speculative_fibers.iter_mut()) + { + fiber.start_time = the_time; + for attempt in &mut fiber.attempts { + attempt.send_time = the_time; + match &mut attempt.result { + Some(AttemptResult::Success(succ_time)) => *succ_time = the_time, + Some(AttemptResult::Error(err_time, _, _)) => *err_time = the_time, + None => {} + } + } + } + } + + history +} + +// Set a single node for all attempts within StructuredHistory. +// When running against real life nodes this address may change, +// setting it to one value makes it possible to run tests consistently. +fn set_one_node(mut history: StructuredHistory) -> StructuredHistory { + let the_node: SocketAddr = node1_addr(); + + for query in &mut history.queries { + for fiber in std::iter::once(&mut query.non_speculative_fiber) + .chain(query.speculative_fibers.iter_mut()) + { + for attempt in &mut fiber.attempts { + attempt.node_addr = the_node; + } + } + } + + history +} + +// Set a single error message for all DbErrors within StructuredHistory. +// The error message changes between Scylla/Cassandra/their versions. +// Setting it to one value makes it possible to run tests consistently. +fn set_one_db_error_message(mut history: StructuredHistory) -> StructuredHistory { + let set_msg = |err: &mut QueryError| { + if let QueryError::DbError(_, msg) = err { + *msg = "Error message from database".to_string(); + } + }; + + for query in &mut history.queries { + if let Some(QueryHistoryResult::Error(_, err)) = &mut query.result { + set_msg(err); + } + for fiber in std::iter::once(&mut query.non_speculative_fiber) + .chain(query.speculative_fibers.iter_mut()) + { + for attempt in &mut fiber.attempts { + if let Some(AttemptResult::Error(_, err, _)) = &mut attempt.result { + set_msg(err); + } + } + } + } + + history +} + +fn node1_addr() -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 19042) +} + +#[tokio::test] +async fn successful_query_history() { + setup_tracing(); + let session = create_new_session_builder().build().await.unwrap(); + + let mut query = Query::new("SELECT * FROM system.local"); + let history_collector = Arc::new(HistoryCollector::new()); + query.set_history_listener(history_collector.clone()); + + session.query_unpaged(query.clone(), ()).await.unwrap(); + + let history: StructuredHistory = history_collector.clone_structured_history(); + + let displayed = "Queries History: +=== Query #0 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Success at 2022-02-22 20:22:22 UTC +| +| Query successful at 2022-02-22 20:22:22 UTC +================= +"; + assert_eq!( + displayed, + format!( + "{}", + set_one_db_error_message(set_one_node(set_one_time(history))) + ) + ); + + // Prepared queries retain the history listener set in Query. + let prepared = session.prepare(query).await.unwrap(); + session.execute_unpaged(&prepared, ()).await.unwrap(); + + let history2: StructuredHistory = history_collector.clone_structured_history(); + + let displayed2 = "Queries History: +=== Query #0 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Success at 2022-02-22 20:22:22 UTC +| +| Query successful at 2022-02-22 20:22:22 UTC +================= +=== Query #1 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Success at 2022-02-22 20:22:22 UTC +| +| Query successful at 2022-02-22 20:22:22 UTC +================= +"; + assert_eq!( + displayed2, + format!( + "{}", + set_one_db_error_message(set_one_node(set_one_time(history2))) + ) + ); +} + +#[tokio::test] +async fn failed_query_history() { + setup_tracing(); + let session = create_new_session_builder().build().await.unwrap(); + + let mut query = Query::new("This isnt even CQL"); + let history_collector = Arc::new(HistoryCollector::new()); + query.set_history_listener(history_collector.clone()); + + assert!(session.query_unpaged(query.clone(), ()).await.is_err()); + + let history: StructuredHistory = history_collector.clone_structured_history(); + + let displayed = +"Queries History: +=== Query #0 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Error at 2022-02-22 20:22:22 UTC +| Error: Database returned an error: The submitted query has a syntax error, Error message: Error message from database +| Retry decision: DontRetry +| +| Query failed at 2022-02-22 20:22:22 UTC +| Error: Database returned an error: The submitted query has a syntax error, Error message: Error message from database +================= +"; + assert_eq!( + displayed, + format!( + "{}", + set_one_db_error_message(set_one_node(set_one_time(history))) + ) + ); +} + +#[tokio::test] +async fn iterator_query_history() { + setup_tracing(); + let session = create_new_session_builder().build().await.unwrap(); + let ks = unique_keyspace_name(); + session + .query_unpaged(format!("CREATE KEYSPACE {} WITH REPLICATION = {{'class' : 'NetworkTopologyStrategy', 'replication_factor' : 1}}", ks), &[]) + .await + .unwrap(); + session.use_keyspace(ks, true).await.unwrap(); + + session + .query_unpaged("CREATE TABLE t (p int primary key)", ()) + .await + .unwrap(); + for i in 0..32 { + session + .query_unpaged("INSERT INTO t (p) VALUES (?)", (i,)) + .await + .unwrap(); + } + + let mut iter_query: Query = Query::new("SELECT * FROM t"); + iter_query.set_page_size(8); + let history_collector = Arc::new(HistoryCollector::new()); + iter_query.set_history_listener(history_collector.clone()); + + let mut rows_iterator = session + .query_iter(iter_query, ()) + .await + .unwrap() + .rows_stream::() + .unwrap(); + while let Some(_row) = rows_iterator.next().await { + // Receive rows... + } + + let history = history_collector.clone_structured_history(); + + assert!(history.queries.len() >= 4); + + let displayed_prefix = "Queries History: +=== Query #0 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Success at 2022-02-22 20:22:22 UTC +| +| Query successful at 2022-02-22 20:22:22 UTC +================= +=== Query #1 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Success at 2022-02-22 20:22:22 UTC +| +| Query successful at 2022-02-22 20:22:22 UTC +================= +=== Query #2 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Success at 2022-02-22 20:22:22 UTC +| +| Query successful at 2022-02-22 20:22:22 UTC +================= +=== Query #3 === +| start_time: 2022-02-22 20:22:22 UTC +| Non-speculative attempts: +| - Attempt #0 sent to 127.0.0.1:19042 +| request send time: 2022-02-22 20:22:22 UTC +| Success at 2022-02-22 20:22:22 UTC +| +| Query successful at 2022-02-22 20:22:22 UTC +================= +"; + let displayed_str = format!( + "{}", + set_one_db_error_message(set_one_node(set_one_time(history))) + ); + + assert!(displayed_str.starts_with(displayed_prefix),); +} diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index 8a9d22db02..24dfe731a6 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -4,6 +4,7 @@ mod cql_collections; mod cql_types; mod cql_value; mod execution_profiles; +mod history; mod hygiene; mod large_batch_statements; mod lwt_optimisation; From 3b2963b3694db50498434e16691b78995c090af4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 15:21:43 +0100 Subject: [PATCH 10/12] default policy: Move some tests to integration --- .../src/transport/load_balancing/default.rs | 25 +----------------- scylla/tests/integration/default_policy.rs | 26 +++++++++++++++++++ scylla/tests/integration/main.rs | 1 + 3 files changed, 28 insertions(+), 24 deletions(-) create mode 100644 scylla/tests/integration/default_policy.rs diff --git a/scylla/src/transport/load_balancing/default.rs b/scylla/src/transport/load_balancing/default.rs index b445dea5fb..fde0578ab0 100644 --- a/scylla/src/transport/load_balancing/default.rs +++ b/scylla/src/transport/load_balancing/default.rs @@ -3128,7 +3128,7 @@ mod latency_awareness { use crate::{ load_balancing::default::NodeLocationPreference, routing::Shard, - test_utils::{create_new_session_builder, setup_tracing}, + test_utils::setup_tracing, transport::locator::test::{TABLE_INVALID, TABLE_NTS_RF_2, TABLE_NTS_RF_3}, }; use crate::{ @@ -3141,7 +3141,6 @@ mod latency_awareness { locator::test::{id_to_invalid_addr, A, B, C, D, E, F, G}, ClusterData, NodeAddr, }, - ExecutionProfile, }; use tokio::time::Instant; @@ -3847,28 +3846,6 @@ mod latency_awareness { } } - // This is a regression test for #696. - #[tokio::test] - #[ntest::timeout(1000)] - async fn latency_aware_query_completes() { - setup_tracing(); - let policy = DefaultPolicy::builder() - .latency_awareness(LatencyAwarenessBuilder::default()) - .build(); - let handle = ExecutionProfile::builder() - .load_balancing_policy(policy) - .build() - .into_handle(); - - let session = create_new_session_builder() - .default_execution_profile_handle(handle) - .build() - .await - .unwrap(); - - session.query_unpaged("whatever", ()).await.unwrap_err(); - } - #[tokio::test(start_paused = true)] async fn timestamped_average_works_when_clock_stops() { setup_tracing(); diff --git a/scylla/tests/integration/default_policy.rs b/scylla/tests/integration/default_policy.rs new file mode 100644 index 0000000000..e097cf418a --- /dev/null +++ b/scylla/tests/integration/default_policy.rs @@ -0,0 +1,26 @@ +use scylla::load_balancing::{DefaultPolicy, LatencyAwarenessBuilder}; +use scylla::ExecutionProfile; + +use crate::utils::{create_new_session_builder, setup_tracing}; + +// This is a regression test for #696. +#[tokio::test] +#[ntest::timeout(1000)] +async fn latency_aware_query_completes() { + setup_tracing(); + let policy = DefaultPolicy::builder() + .latency_awareness(LatencyAwarenessBuilder::default()) + .build(); + let handle = ExecutionProfile::builder() + .load_balancing_policy(policy) + .build() + .into_handle(); + + let session = create_new_session_builder() + .default_execution_profile_handle(handle) + .build() + .await + .unwrap(); + + session.query_unpaged("whatever", ()).await.unwrap_err(); +} diff --git a/scylla/tests/integration/main.rs b/scylla/tests/integration/main.rs index 24dfe731a6..d510dc6fd7 100644 --- a/scylla/tests/integration/main.rs +++ b/scylla/tests/integration/main.rs @@ -3,6 +3,7 @@ mod consistency; mod cql_collections; mod cql_types; mod cql_value; +mod default_policy; mod execution_profiles; mod history; mod hygiene; From f87e1338de214b4f90ff388e2da0ec804e32ad53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Wed, 20 Nov 2024 15:55:41 +0100 Subject: [PATCH 11/12] Tests: Change the way tablet support is checked Fixes: https://github.com/scylladb/scylla-rust-driver/issues/1048 See the issue description for more information. --- scylla/src/utils/test_utils.rs | 15 +-------------- scylla/tests/integration/utils.rs | 15 +-------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/scylla/src/utils/test_utils.rs b/scylla/src/utils/test_utils.rs index 68035fcee3..d745a31359 100644 --- a/scylla/src/utils/test_utils.rs +++ b/scylla/src/utils/test_utils.rs @@ -1,6 +1,5 @@ use crate::transport::session_builder::{GenericSessionBuilder, SessionBuilderKind}; use crate::Session; -use scylla_cql::frame::response::result::Row; use std::{num::NonZeroU32, time::Duration}; use std::{ sync::atomic::{AtomicUsize, Ordering}, @@ -92,19 +91,7 @@ pub(crate) fn create_new_session_builder() -> GenericSessionBuilder bool { - let result = session - .query_unpaged( - "select column_name from system_schema.columns where - keyspace_name = 'system_schema' - and table_name = 'scylla_keyspaces' - and column_name = 'initial_tablets'", - &[], - ) - .await - .unwrap() - .into_rows_result(); - - result.is_ok_and(|rows_result| rows_result.single_row::().is_ok()) + supports_feature(session, "TABLETS").await } pub(crate) fn setup_tracing() { diff --git a/scylla/tests/integration/utils.rs b/scylla/tests/integration/utils.rs index 4ec8539c5c..ecbffcb1dc 100644 --- a/scylla/tests/integration/utils.rs +++ b/scylla/tests/integration/utils.rs @@ -1,6 +1,5 @@ use futures::Future; use scylla::deserialize::DeserializeValue; -use scylla::frame::response::result::Row; use scylla::transport::session_builder::{GenericSessionBuilder, SessionBuilderKind}; use scylla::Session; use std::collections::HashMap; @@ -123,19 +122,7 @@ pub(crate) async fn supports_feature(session: &Session, feature: &str) -> bool { #[allow(unused)] pub(crate) async fn scylla_supports_tablets(session: &Session) -> bool { - let result = session - .query_unpaged( - "select column_name from system_schema.columns where - keyspace_name = 'system_schema' - and table_name = 'scylla_keyspaces' - and column_name = 'initial_tablets'", - &[], - ) - .await - .unwrap() - .into_rows_result(); - - result.is_ok_and(|rows_result| rows_result.single_row::().is_ok()) + supports_feature(session, "TABLETS").await } // Creates a generic session builder based on conditional compilation configuration From 2b5a0848eea6b3a59dfe1d616723b517e309d035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Mon, 4 Nov 2024 14:01:48 +0100 Subject: [PATCH 12/12] integration/utils: Remove unnecessary annotations --- scylla/tests/integration/utils.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scylla/tests/integration/utils.rs b/scylla/tests/integration/utils.rs index ecbffcb1dc..8e187cbe8e 100644 --- a/scylla/tests/integration/utils.rs +++ b/scylla/tests/integration/utils.rs @@ -12,7 +12,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use scylla_proxy::{Node, Proxy, ProxyError, RunningProxy, ShardAwareness}; -#[cfg(test)] pub(crate) fn setup_tracing() { let _ = tracing_subscriber::fmt::fmt() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) @@ -22,7 +21,6 @@ pub(crate) fn setup_tracing() { static UNIQUE_COUNTER: AtomicUsize = AtomicUsize::new(0); -#[allow(unused)] pub(crate) fn unique_keyspace_name() -> String { let cnt = UNIQUE_COUNTER.fetch_add(1, Ordering::SeqCst); let name = format!( @@ -87,7 +85,6 @@ where running_proxy.finish().await } -#[allow(unused)] pub(crate) async fn supports_feature(session: &Session, feature: &str) -> bool { // Cassandra doesn't have a concept of features, so first detect // if there is the `supported_features` column in system.local @@ -120,7 +117,6 @@ pub(crate) async fn supports_feature(session: &Session, feature: &str) -> bool { .any(|f| f == feature) } -#[allow(unused)] pub(crate) async fn scylla_supports_tablets(session: &Session) -> bool { supports_feature(session, "TABLETS").await } @@ -128,7 +124,6 @@ pub(crate) async fn scylla_supports_tablets(session: &Session) -> bool { // Creates a generic session builder based on conditional compilation configuration // For SessionBuilder of DefaultMode type, adds localhost to known hosts, as all of the tests // connect to localhost. -#[allow(unused)] pub(crate) fn create_new_session_builder() -> GenericSessionBuilder { let session_builder = { #[cfg(not(scylla_cloud_tests))]