diff --git a/Cargo.lock b/Cargo.lock index 1721d665..682bdf26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1385,7 +1385,7 @@ dependencies = [ [[package]] name = "divviup-api" -version = "0.3.22" +version = "0.4.0" dependencies = [ "aes-gcm", "async-lock 3.4.0", @@ -1453,7 +1453,7 @@ dependencies = [ [[package]] name = "divviup-cli" -version = "0.3.22" +version = "0.4.0" dependencies = [ "anyhow", "base64 0.22.1", @@ -1483,7 +1483,7 @@ dependencies = [ [[package]] name = "divviup-client" -version = "0.3.22" +version = "0.4.0" dependencies = [ "base64 0.22.1", "divviup-api", @@ -1493,6 +1493,8 @@ dependencies = [ "futures-lite 2.3.0", "janus_messages", "log", + "num-bigint", + "num-rational", "pad-adapter", "prio", "serde", @@ -2745,7 +2747,7 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "migration" -version = "0.3.22" +version = "0.4.0" dependencies = [ "async-std", "clap", @@ -4887,7 +4889,7 @@ dependencies = [ [[package]] name = "test-support" -version = "0.3.22" +version = "0.4.0" dependencies = [ "base64 0.22.1", "divviup-api", diff --git a/Cargo.toml b/Cargo.toml index 113e5030..dfcc52d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,15 +2,15 @@ members = [".", "migration", "client", "test-support", "cli"] [workspace.package] -version = "0.3.22" +version = "0.4.0" edition = "2021" license = "MPL-2.0" homepage = "https://divviup.org" repository = "https://github.com/divviup/divviup-api" [workspace.dependencies] -divviup-client = { path = "./client", version = "0.3.22" } -divviup-cli = { path = "./cli", version = "0.3.22" } +divviup-client = { path = "./client", version = "0.4.0" } +divviup-cli = { path = "./cli", version = "0.4.0" } divviup-api.path = "." test-support.path = "./test-support" diff --git a/cli/src/tasks.rs b/cli/src/tasks.rs index c3822977..9dc388d9 100644 --- a/cli/src/tasks.rs +++ b/cli/src/tasks.rs @@ -1,6 +1,9 @@ use crate::{CliResult, DetermineAccountId, Error, Output}; use clap::Subcommand; -use divviup_client::{DivviupClient, Histogram, NewTask, SumVec, Uuid, Vdaf}; +use divviup_client::{ + dp_strategy::{self, PureDpBudget, PureDpDiscreteLaplace}, + BigUint, DivviupClient, Histogram, NewTask, Ratio, SumVec, Uuid, Vdaf, +}; use humantime::{Duration, Timestamp}; use std::time::SystemTime; use time::OffsetDateTime; @@ -14,6 +17,11 @@ pub enum VdafName { SumVec, } +#[derive(clap::ValueEnum, Clone, Debug)] +pub enum DpStrategy { + PureDpDiscreteLaplace, +} + #[derive(Subcommand, Debug)] pub enum TaskAction { /// list all tasks for the target account @@ -52,6 +60,10 @@ pub enum TaskAction { bits: Option, #[arg(long)] chunk_length: Option, + #[arg(long, requires = "differential_privacy_epsilon")] + differential_privacy_strategy: Option, + #[arg(long, requires = "differential_privacy_strategy")] + differential_privacy_epsilon: Option, }, /// rename a task @@ -108,24 +120,65 @@ impl TaskAction { length, bits, chunk_length, + differential_privacy_strategy, + differential_privacy_epsilon, } => { let vdaf = match vdaf { - VdafName::Count => Vdaf::Count, + VdafName::Count => { + if differential_privacy_strategy.is_some() + || differential_privacy_epsilon.is_some() + { + return Err(Error::Other( + "differential privacy noise is not yet supported with Prio3Count" + .into(), + )); + } + Vdaf::Count + } VdafName::Histogram => { + let dp_strategy = + match (differential_privacy_strategy, differential_privacy_epsilon) { + (None, None) => dp_strategy::Prio3Histogram::NoDifferentialPrivacy, + (None, Some(_)) => { + return Err(Error::Other( + "missing differential-privacy-strategy".into(), + )) + } + (Some(_), None) => { + return Err(Error::Other( + "missing differential-privacy-epsilon".into(), + )) + } + (Some(DpStrategy::PureDpDiscreteLaplace), Some(epsilon)) => { + dp_strategy::Prio3Histogram::PureDpDiscreteLaplace( + PureDpDiscreteLaplace { + budget: PureDpBudget { + epsilon: float_to_biguint_ratio(epsilon) + .ok_or_else(|| { + Error::Other("invalid epsilon".into()) + })?, + }, + }, + ) + } + }; match (length, categorical_buckets, continuous_buckets) { (Some(length), None, None) => Vdaf::Histogram(Histogram::Length { length, chunk_length, + dp_strategy, }), (None, Some(buckets), None) => { Vdaf::Histogram(Histogram::Categorical { buckets, chunk_length, + dp_strategy, }) } (None, None, Some(buckets)) => Vdaf::Histogram(Histogram::Continuous { buckets, chunk_length, + dp_strategy, }), (None, None, None) => { return Err(Error::Other("continuous-buckets, categorical-buckets, or length are required for histogram vdaf".into())); @@ -135,15 +188,66 @@ impl TaskAction { } } } - VdafName::Sum => Vdaf::Sum { - bits: bits.unwrap(), - }, - VdafName::CountVec => Vdaf::CountVec { - length: length.unwrap(), - chunk_length, - }, + VdafName::Sum => { + if differential_privacy_strategy.is_some() + || differential_privacy_epsilon.is_some() + { + return Err(Error::Other( + "differential privacy noise is not yet supported with Prio3Sum" + .into(), + )); + } + Vdaf::Sum { + bits: bits.unwrap(), + } + } + VdafName::CountVec => { + if differential_privacy_strategy.is_some() + || differential_privacy_epsilon.is_some() + { + return Err(Error::Other( + "differential privacy noise is not supported with Prio3CountVec" + .into(), + )); + } + Vdaf::CountVec { + length: length.unwrap(), + chunk_length, + } + } VdafName::SumVec => { - Vdaf::SumVec(SumVec::new(bits.unwrap(), length.unwrap(), chunk_length)) + let dp_strategy = + match (differential_privacy_strategy, differential_privacy_epsilon) { + (None, None) => dp_strategy::Prio3SumVec::NoDifferentialPrivacy, + (None, Some(_)) => { + return Err(Error::Other( + "missing differential-privacy-strategy".into(), + )) + } + (Some(_), None) => { + return Err(Error::Other( + "missing differential-privacy-epsilon".into(), + )) + } + (Some(DpStrategy::PureDpDiscreteLaplace), Some(epsilon)) => { + dp_strategy::Prio3SumVec::PureDpDiscreteLaplace( + PureDpDiscreteLaplace { + budget: PureDpBudget { + epsilon: float_to_biguint_ratio(epsilon) + .ok_or_else(|| { + Error::Other("invalid epsilon".into()) + })?, + }, + }, + ) + } + }; + Vdaf::SumVec(SumVec::new( + bits.unwrap(), + length.unwrap(), + chunk_length, + dp_strategy, + )) } }; @@ -201,3 +305,12 @@ impl TaskAction { Ok(()) } } + +fn float_to_biguint_ratio(value: f64) -> Option> { + let signed_ratio = Ratio::from_float(value)?; + let unsigned_ratio = Ratio::new( + signed_ratio.numer().clone().try_into().ok()?, + signed_ratio.denom().clone().try_into().ok()?, + ); + Some(unsigned_ratio) +} diff --git a/client/Cargo.toml b/client/Cargo.toml index a744afba..b6f0556c 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -27,6 +27,8 @@ time = { version = "0.3.36", features = ["serde", "serde-well-known"] } log = "0.4.22" pad-adapter = "0.1.1" janus_messages = "0.7.25" +num-bigint = "0.4.6" +num-rational = "0.4.2" [dev-dependencies] divviup-api.workspace = true diff --git a/client/src/dp_strategy.rs b/client/src/dp_strategy.rs new file mode 100644 index 00000000..c6516722 --- /dev/null +++ b/client/src/dp_strategy.rs @@ -0,0 +1,31 @@ +use num_bigint::BigUint; +use num_rational::Ratio; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Default)] +#[serde(tag = "dp_strategy")] +#[non_exhaustive] +pub enum Prio3Histogram { + #[default] + NoDifferentialPrivacy, + PureDpDiscreteLaplace(PureDpDiscreteLaplace), +} + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Default)] +#[serde(tag = "dp_strategy")] +#[non_exhaustive] +pub enum Prio3SumVec { + #[default] + NoDifferentialPrivacy, + PureDpDiscreteLaplace(PureDpDiscreteLaplace), +} + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +pub struct PureDpDiscreteLaplace { + pub budget: PureDpBudget, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +pub struct PureDpBudget { + pub epsilon: Ratio, +} diff --git a/client/src/lib.rs b/client/src/lib.rs index 7c2d5096..1937619c 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -13,6 +13,7 @@ mod account; mod aggregator; mod api_token; mod collector_credentials; +pub mod dp_strategy; mod membership; mod protocol; mod task; @@ -38,6 +39,8 @@ pub use janus_messages::{ HpkeConfig, HpkePublicKey, }; pub use membership::Membership; +pub use num_bigint::BigUint; +pub use num_rational::Ratio; pub use protocol::Protocol; pub use task::{Histogram, NewTask, SumVec, Task, Vdaf}; pub use time::OffsetDateTime; diff --git a/client/src/task.rs b/client/src/task.rs index 6ff2b40a..499ff6f1 100644 --- a/client/src/task.rs +++ b/client/src/task.rs @@ -3,6 +3,8 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; +use crate::dp_strategy; + #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] pub struct Task { pub id: String, @@ -79,14 +81,20 @@ pub enum Histogram { Categorical { buckets: Vec, chunk_length: Option, + #[serde(default)] + dp_strategy: dp_strategy::Prio3Histogram, }, Continuous { buckets: Vec, chunk_length: Option, + #[serde(default)] + dp_strategy: dp_strategy::Prio3Histogram, }, Length { length: u64, chunk_length: Option, + #[serde(default)] + dp_strategy: dp_strategy::Prio3Histogram, }, } @@ -110,22 +118,39 @@ impl Histogram { .map(|c| c as usize) .unwrap_or_else(|| optimal_chunk_length(self.length())) } + + /// The differential privacy strategy. + pub fn dp_strategy(&self) -> &dp_strategy::Prio3Histogram { + match self { + Histogram::Categorical { dp_strategy, .. } + | Histogram::Continuous { dp_strategy, .. } + | Histogram::Length { dp_strategy, .. } => dp_strategy, + } + } } -#[derive(Serialize, Deserialize, Debug, Clone, Copy, Eq, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] pub struct SumVec { pub bits: u8, pub length: u64, chunk_length: Option, + #[serde(default)] + pub dp_strategy: dp_strategy::Prio3SumVec, } impl SumVec { /// Create a new SumVec - pub fn new(bits: u8, length: u64, chunk_length: Option) -> Self { + pub fn new( + bits: u8, + length: u64, + chunk_length: Option, + dp_strategy: dp_strategy::Prio3SumVec, + ) -> Self { Self { bits, length, chunk_length, + dp_strategy, } } diff --git a/documentation/openapi.yml b/documentation/openapi.yml index 39638944..447fe661 100644 --- a/documentation/openapi.yml +++ b/documentation/openapi.yml @@ -13,7 +13,7 @@ info: license: name: MPL-2.0 url: https://www.mozilla.org/en-US/MPL/2.0/ - version: "0.1" + version: "0.4" externalDocs: description: The Divvi Up API repository url: https://github.com/divviup/divviup-api @@ -805,6 +805,25 @@ components: type: string chunk_length: type: number + dp_strategy: + type: object + properties: + dp_strategy: + type: string + enum: [NoDifferentialPrivacy, PureDpDiscreteLaplace] + budget: + type: object + properties: + epsilon: + type: array + minItems: 2 + maxItems: 2 + items: + type: array + items: + type: number + minimum: 0 + maximum: 4294967295 required: [type] ApiToken: type: object diff --git a/src/clients/aggregator_client/api_types.rs b/src/clients/aggregator_client/api_types.rs index c842e53d..c10cf764 100644 --- a/src/clients/aggregator_client/api_types.rs +++ b/src/clients/aggregator_client/api_types.rs @@ -3,7 +3,10 @@ use crate::{ aggregator::{ Features, QueryTypeName, QueryTypeNameSet, Role as AggregatorRole, VdafNameSet, }, - task::vdaf::{BucketLength, ContinuousBuckets, CountVec, Histogram, Sum, SumVec, Vdaf}, + task::vdaf::{ + BucketLength, ContinuousBuckets, CountVec, DpBudget, DpStrategy, DpStrategyKind, + Histogram, Sum, SumVec, Vdaf, + }, Aggregator, Protocol, ProvisionableTask, Task, }, handler::Error, @@ -18,6 +21,8 @@ pub use janus_messages::{ HpkeKemId, HpkePublicKey, Role, TaskId, Time as JanusTime, }; +pub mod dp_strategies; + #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] #[non_exhaustive] pub enum AggregatorVdaf { @@ -34,80 +39,22 @@ pub enum AggregatorVdaf { bits: u8, length: u64, chunk_length: Option, + dp_strategy: dp_strategies::Prio3SumVec, }, } -impl PartialEq for AggregatorVdaf { - fn eq(&self, other: &Vdaf) -> bool { - other.eq(self) - } -} - -impl PartialEq for Vdaf { - fn eq(&self, other: &AggregatorVdaf) -> bool { - match (self, other) { - (Vdaf::Count, AggregatorVdaf::Prio3Count) => true, - ( - Vdaf::Histogram(histogram), - AggregatorVdaf::Prio3Histogram(HistogramType::Opaque { - length, - chunk_length, - }), - ) => histogram.length() == *length && histogram.chunk_length() == *chunk_length, - ( - Vdaf::Histogram(Histogram::Continuous(ContinuousBuckets { - buckets: Some(lhs_buckets), - chunk_length: lhs_chunk_length, - })), - AggregatorVdaf::Prio3Histogram(HistogramType::Buckets { - buckets: rhs_buckets, - chunk_length: rhs_chunk_length, - }), - ) => lhs_buckets == rhs_buckets && lhs_chunk_length == rhs_chunk_length, - (Vdaf::Sum(Sum { bits: Some(lhs) }), AggregatorVdaf::Prio3Sum { bits: rhs }) => { - lhs == rhs - } - ( - Vdaf::CountVec(CountVec { - length: Some(lhs_length), - chunk_length: lhs_chunk_length, - }), - AggregatorVdaf::Prio3CountVec { - length: rhs_length, - chunk_length: rhs_chunk_length, - }, - ) => lhs_length == rhs_length && lhs_chunk_length == rhs_chunk_length, - ( - Vdaf::SumVec(SumVec { - bits: Some(lhs_bits), - length: Some(lhs_length), - chunk_length: lhs_chunk_length, - }), - AggregatorVdaf::Prio3SumVec { - bits: rhs_bits, - length: rhs_length, - chunk_length: rhs_chunk_length, - }, - ) => { - lhs_bits == rhs_bits - && lhs_length == rhs_length - && lhs_chunk_length == rhs_chunk_length - } - _ => false, - } - } -} - #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] #[serde(untagged)] pub enum HistogramType { Opaque { length: u64, chunk_length: Option, + dp_strategy: dp_strategies::Prio3Histogram, }, Buckets { buckets: Vec, chunk_length: Option, + dp_strategy: dp_strategies::Prio3Histogram, }, } @@ -119,17 +66,53 @@ impl From for Vdaf { AggregatorVdaf::Prio3Histogram(HistogramType::Buckets { buckets, chunk_length, - }) => Self::Histogram(Histogram::Continuous(ContinuousBuckets { - buckets: Some(buckets), - chunk_length, - })), + dp_strategy, + }) => { + let dp_strategy = match dp_strategy { + dp_strategies::Prio3Histogram::NoDifferentialPrivacy => DpStrategy { + dp_strategy: DpStrategyKind::NoDifferentialPrivacy, + budget: DpBudget { epsilon: None }, + }, + dp_strategies::Prio3Histogram::PureDpDiscreteLaplace(dp_strategy) => { + DpStrategy { + dp_strategy: DpStrategyKind::PureDpDiscreteLaplace, + budget: DpBudget { + epsilon: Some(dp_strategy.budget.epsilon.to_vec()), + }, + } + } + }; + Self::Histogram(Histogram::Continuous(ContinuousBuckets { + buckets: Some(buckets), + chunk_length, + dp_strategy, + })) + } AggregatorVdaf::Prio3Histogram(HistogramType::Opaque { length, chunk_length, - }) => Self::Histogram(Histogram::Opaque(BucketLength { - length, - chunk_length, - })), + dp_strategy, + }) => { + let dp_strategy = match dp_strategy { + dp_strategies::Prio3Histogram::NoDifferentialPrivacy => DpStrategy { + dp_strategy: DpStrategyKind::NoDifferentialPrivacy, + budget: DpBudget { epsilon: None }, + }, + dp_strategies::Prio3Histogram::PureDpDiscreteLaplace(dp_strategy) => { + DpStrategy { + dp_strategy: DpStrategyKind::PureDpDiscreteLaplace, + budget: DpBudget { + epsilon: Some(dp_strategy.budget.epsilon.to_vec()), + }, + } + } + }; + Self::Histogram(Histogram::Opaque(BucketLength { + length, + chunk_length, + dp_strategy, + })) + } AggregatorVdaf::Prio3CountVec { length, chunk_length, @@ -141,11 +124,27 @@ impl From for Vdaf { bits, length, chunk_length, - } => Self::SumVec(SumVec { - length: Some(length), - bits: Some(bits), - chunk_length, - }), + dp_strategy, + } => { + let dp_strategy = match dp_strategy { + dp_strategies::Prio3SumVec::NoDifferentialPrivacy => DpStrategy { + dp_strategy: DpStrategyKind::NoDifferentialPrivacy, + budget: DpBudget { epsilon: None }, + }, + dp_strategies::Prio3SumVec::PureDpDiscreteLaplace(dp_strategy) => DpStrategy { + dp_strategy: DpStrategyKind::PureDpDiscreteLaplace, + budget: DpBudget { + epsilon: Some(dp_strategy.budget.epsilon.to_vec()), + }, + }, + }; + Self::SumVec(SumVec { + length: Some(length), + bits: Some(bits), + chunk_length, + dp_strategy, + }) + } } } } diff --git a/src/clients/aggregator_client/api_types/dp_strategies.rs b/src/clients/aggregator_client/api_types/dp_strategies.rs new file mode 100644 index 00000000..5bbf8e5f --- /dev/null +++ b/src/clients/aggregator_client/api_types/dp_strategies.rs @@ -0,0 +1,25 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "dp_strategy")] +pub enum Prio3Histogram { + NoDifferentialPrivacy, + PureDpDiscreteLaplace(PureDpDiscreteLaplace), +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "dp_strategy")] +pub enum Prio3SumVec { + NoDifferentialPrivacy, + PureDpDiscreteLaplace(PureDpDiscreteLaplace), +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PureDpDiscreteLaplace { + pub budget: PureDpBudget, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PureDpBudget { + pub epsilon: [Vec; 2], +} diff --git a/src/entity/aggregator/feature.rs b/src/entity/aggregator/feature.rs index 1f59f1ee..1f01ca11 100644 --- a/src/entity/aggregator/feature.rs +++ b/src/entity/aggregator/feature.rs @@ -6,6 +6,7 @@ pub enum Feature { TokenHash, UploadMetrics, TimeBucketedFixedSize, + PureDpDiscreteLaplace, #[serde(untagged)] Unknown(String), } diff --git a/src/entity/task/new_task.rs b/src/entity/task/new_task.rs index d97dafbf..ae2958de 100644 --- a/src/entity/task/new_task.rs +++ b/src/entity/task/new_task.rs @@ -12,6 +12,7 @@ use rand::Rng; use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter}; use sha2::{Digest, Sha256}; use validator::{ValidationErrors, ValidationErrorsKind}; +use vdaf::{DpStrategy, DpStrategyKind, SumVec}; #[derive(Deserialize, Validate, Debug, Clone, Default)] pub struct NewTask { @@ -227,6 +228,38 @@ impl NewTask { ) } + let uses_pure_dp_discrete_laplace = match &self.vdaf { + Some(Vdaf::SumVec(SumVec { + dp_strategy: + DpStrategy { + dp_strategy: DpStrategyKind::PureDpDiscreteLaplace, + .. + }, + .. + })) => true, + Some(Vdaf::Histogram(histogram)) => matches!( + histogram.dp_strategy().dp_strategy, + DpStrategyKind::PureDpDiscreteLaplace + ), + _ => false, + }; + if uses_pure_dp_discrete_laplace + && !leader.features.contains(&Feature::PureDpDiscreteLaplace) + { + errors.add( + "leader_aggregator_id", + ValidationError::new("pure-dp-discrete-laplace-unsupported"), + ); + } + if uses_pure_dp_discrete_laplace + && !helper.features.contains(&Feature::PureDpDiscreteLaplace) + { + errors.add( + "helper_aggregator_id", + ValidationError::new("pure-dp-discrete-laplace-unsupported"), + ); + } + if errors.is_empty() { Some((leader, helper, resolved_protocol)) } else { diff --git a/src/entity/task/vdaf.rs b/src/entity/task/vdaf.rs index 8bf631d8..cb991144 100644 --- a/src/entity/task/vdaf.rs +++ b/src/entity/task/vdaf.rs @@ -1,5 +1,8 @@ use crate::{ - clients::aggregator_client::api_types::{AggregatorVdaf, HistogramType}, + clients::aggregator_client::api_types::{ + dp_strategies::{self, PureDpBudget, PureDpDiscreteLaplace}, + AggregatorVdaf, HistogramType, + }, entity::{aggregator::VdafName, Protocol}, }; use prio::vdaf::prio3::optimal_chunk_length; @@ -47,11 +50,20 @@ impl Histogram { Ok(AggregatorVdaf::Prio3Histogram(HistogramType::Opaque { length: self.length(), chunk_length: Some(chunk_length), + dp_strategy: self.dp_strategy().representation_histogram(), })) } else { panic!("chunk_length was not populated"); } } + + pub fn dp_strategy(&self) -> &DpStrategy { + match self { + Histogram::Opaque(BucketLength { dp_strategy, .. }) + | Histogram::Categorical(CategoricalBuckets { dp_strategy, .. }) + | Histogram::Continuous(ContinuousBuckets { dp_strategy, .. }) => dp_strategy, + } + } } #[derive(Serialize, Deserialize, Validate, Debug, Clone, Eq, PartialEq)] @@ -66,6 +78,10 @@ pub struct ContinuousBuckets { #[validate(range(min = 1))] pub chunk_length: Option, + + #[serde(default)] + #[validate(nested, custom(function = "validate_dp_strategy"))] + pub dp_strategy: DpStrategy, } #[derive(Serialize, Deserialize, Validate, Debug, Clone, Eq, PartialEq)] @@ -75,15 +91,95 @@ pub struct CategoricalBuckets { #[validate(range(min = 1))] pub chunk_length: Option, + + #[serde(default)] + #[validate(nested, custom(function = "validate_dp_strategy"))] + pub dp_strategy: DpStrategy, } -#[derive(Serialize, Deserialize, Validate, Debug, Clone, Eq, PartialEq, Copy)] +#[derive(Serialize, Deserialize, Validate, Debug, Clone, Eq, PartialEq)] pub struct BucketLength { #[validate(range(min = 1))] pub length: u64, #[validate(range(min = 1))] pub chunk_length: Option, + + #[serde(default)] + #[validate(nested, custom(function = "validate_dp_strategy"))] + pub dp_strategy: DpStrategy, +} + +#[derive(Serialize, Deserialize, Validate, Debug, Clone, Eq, PartialEq, Default)] +pub struct DpStrategy { + #[serde(default)] + pub dp_strategy: DpStrategyKind, + + #[serde(default)] + #[validate(nested)] + pub budget: DpBudget, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Eq, PartialEq, Default)] +pub enum DpStrategyKind { + #[default] + NoDifferentialPrivacy, + PureDpDiscreteLaplace, +} + +#[derive(Serialize, Deserialize, Validate, Debug, Clone, Eq, PartialEq, Default)] +pub struct DpBudget { + #[validate(length(equal = 2))] + pub epsilon: Option>>, +} + +impl DpStrategy { + fn representation_histogram(&self) -> dp_strategies::Prio3Histogram { + match (self.dp_strategy, &self.budget.epsilon) { + (DpStrategyKind::NoDifferentialPrivacy, None) => { + dp_strategies::Prio3Histogram::NoDifferentialPrivacy + } + (DpStrategyKind::NoDifferentialPrivacy, Some(_)) + | (DpStrategyKind::PureDpDiscreteLaplace, None) => panic!("invalid dp strategy"), + (DpStrategyKind::PureDpDiscreteLaplace, Some(epsilon)) => { + dp_strategies::Prio3Histogram::PureDpDiscreteLaplace(PureDpDiscreteLaplace { + budget: PureDpBudget { + epsilon: epsilon.clone().try_into().expect("invalid epsilon"), + }, + }) + } + } + } + + fn representation_sumvec(&self) -> dp_strategies::Prio3SumVec { + match (self.dp_strategy, &self.budget.epsilon) { + (DpStrategyKind::NoDifferentialPrivacy, None) => { + dp_strategies::Prio3SumVec::NoDifferentialPrivacy + } + (DpStrategyKind::NoDifferentialPrivacy, Some(_)) + | (DpStrategyKind::PureDpDiscreteLaplace, None) => panic!("invalid dp strategy"), + (DpStrategyKind::PureDpDiscreteLaplace, Some(epsilon)) => { + dp_strategies::Prio3SumVec::PureDpDiscreteLaplace(PureDpDiscreteLaplace { + budget: PureDpBudget { + epsilon: epsilon.clone().try_into().expect("invalid epsilon"), + }, + }) + } + } + } +} + +fn validate_dp_strategy(dp_strategy: &DpStrategy) -> Result<(), ValidationError> { + match (dp_strategy.dp_strategy, &dp_strategy.budget.epsilon) { + (DpStrategyKind::NoDifferentialPrivacy, None) => Ok(()), + (DpStrategyKind::NoDifferentialPrivacy, Some(_)) => { + Err(ValidationError::new("extra_epsilon")) + } + (DpStrategyKind::PureDpDiscreteLaplace, None) => { + Err(ValidationError::new("missing_epsilon")) + } + (DpStrategyKind::PureDpDiscreteLaplace, Some(_)) => Ok(()), + } } fn unique(buckets: &[T]) -> Result<(), ValidationError> { @@ -124,7 +220,7 @@ pub struct CountVec { pub chunk_length: Option, } -#[derive(Serialize, Deserialize, Validate, Debug, Clone, Copy, Eq, PartialEq)] +#[derive(Serialize, Deserialize, Validate, Debug, Clone, Eq, PartialEq)] pub struct SumVec { #[validate(required)] pub bits: Option, @@ -134,6 +230,10 @@ pub struct SumVec { #[validate(range(min = 1))] pub chunk_length: Option, + + #[serde(default)] + #[validate(nested, custom(function = "validate_dp_strategy"))] + pub dp_strategy: DpStrategy, } #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] @@ -182,10 +282,12 @@ impl Vdaf { length: Some(length), bits: Some(bits), chunk_length, + dp_strategy, }) => Ok(AggregatorVdaf::Prio3SumVec { bits: *bits, length: *length, chunk_length: *chunk_length, + dp_strategy: dp_strategy.representation_sumvec(), }), Self::CountVec(CountVec { length: Some(length), @@ -243,6 +345,7 @@ impl Vdaf { bits: Some(bits), length: Some(length), chunk_length: chunk_length @ None, + dp_strategy: _, }) => { *chunk_length = Some(optimal_chunk_length(*bits as usize * *length as usize) as u64) } @@ -289,6 +392,7 @@ mod tests { assert!(ContinuousBuckets { buckets: Some(vec![0, 1, 2]), chunk_length: None, + dp_strategy: DpStrategy::default(), } .validate() .is_ok()); @@ -297,6 +401,7 @@ mod tests { ContinuousBuckets { buckets: Some(vec![0, 2, 1]), chunk_length: None, + dp_strategy: DpStrategy::default(), }, "buckets", &["sorted"], @@ -306,6 +411,7 @@ mod tests { ContinuousBuckets { buckets: Some(vec![0, 0, 2]), chunk_length: None, + dp_strategy: DpStrategy::default(), }, "buckets", &["unique"], @@ -317,6 +423,7 @@ mod tests { assert!(CategoricalBuckets { buckets: Some(vec!["a".into(), "b".into()]), chunk_length: None, + dp_strategy: DpStrategy::default(), } .validate() .is_ok()); @@ -325,6 +432,7 @@ mod tests { CategoricalBuckets { buckets: Some(vec!["a".into(), "a".into()]), chunk_length: None, + dp_strategy: DpStrategy::default(), }, "buckets", &["unique"], diff --git a/src/entity/task/vdaf/tests/serde.rs b/src/entity/task/vdaf/tests/serde.rs index e75024e1..0ffed4fb 100644 --- a/src/entity/task/vdaf/tests/serde.rs +++ b/src/entity/task/vdaf/tests/serde.rs @@ -1,5 +1,6 @@ use crate::entity::task::vdaf::{ - BucketLength, CategoricalBuckets, ContinuousBuckets, CountVec, Histogram, Sum, SumVec, Vdaf, + BucketLength, CategoricalBuckets, ContinuousBuckets, CountVec, DpBudget, DpStrategy, + DpStrategyKind, Histogram, Sum, SumVec, Vdaf, }; #[test] @@ -11,6 +12,7 @@ fn json_vdaf() { Vdaf::Histogram(Histogram::Categorical(CategoricalBuckets { buckets: Some(Vec::from(["A".to_owned(), "B".to_owned()])), chunk_length: None, + dp_strategy: DpStrategy::default(), })), ), ( @@ -18,6 +20,7 @@ fn json_vdaf() { Vdaf::Histogram(Histogram::Categorical(CategoricalBuckets { buckets: Some(Vec::from(["A".to_owned(), "B".to_owned()])), chunk_length: Some(2), + dp_strategy: DpStrategy::default(), })), ), ( @@ -25,6 +28,7 @@ fn json_vdaf() { Vdaf::Histogram(Histogram::Continuous(ContinuousBuckets { buckets: Some(Vec::from([1, 10, 100])), chunk_length: None, + dp_strategy: DpStrategy::default(), })), ), ( @@ -32,6 +36,7 @@ fn json_vdaf() { Vdaf::Histogram(Histogram::Continuous(ContinuousBuckets { buckets: Some(Vec::from([1, 10, 100])), chunk_length: Some(2), + dp_strategy: DpStrategy::default(), })), ), ( @@ -39,6 +44,7 @@ fn json_vdaf() { Vdaf::Histogram(Histogram::Opaque(BucketLength { length: 5, chunk_length: None, + dp_strategy: DpStrategy::default(), })), ), ( @@ -46,6 +52,7 @@ fn json_vdaf() { Vdaf::Histogram(Histogram::Opaque(BucketLength { length: 5, chunk_length: Some(2), + dp_strategy: DpStrategy::default(), })), ), ( @@ -72,6 +79,7 @@ fn json_vdaf() { bits: Some(8), length: Some(10), chunk_length: None, + dp_strategy: DpStrategy::default(), }), ), ( @@ -80,9 +88,35 @@ fn json_vdaf() { bits: Some(8), length: Some(10), chunk_length: Some(12), + dp_strategy: DpStrategy::default(), }), ), (r#"{"type":"wrong"}"#, Vdaf::Unrecognized), + ( + r#"{"type":"histogram","length":4,"chunk_length":2,"dp_strategy":{"dp_strategy":"NoDifferentialPrivacy"}}"#, + Vdaf::Histogram(Histogram::Opaque(BucketLength { + length: 4, + chunk_length: Some(2), + dp_strategy: DpStrategy { + dp_strategy: DpStrategyKind::NoDifferentialPrivacy, + budget: DpBudget { epsilon: None }, + }, + })), + ), + ( + r#"{"type":"sum_vec","bits":2,"length":8,"chunk_length":4,"dp_strategy":{"dp_strategy":"PureDpDiscreteLaplace","budget":{"epsilon":[[1],[1]]}}}"#, + Vdaf::SumVec(SumVec { + bits: Some(2), + length: Some(8), + chunk_length: Some(4), + dp_strategy: DpStrategy { + dp_strategy: DpStrategyKind::PureDpDiscreteLaplace, + budget: DpBudget { + epsilon: Some(Vec::from([Vec::from([1]), Vec::from([1])])), + }, + }, + }), + ), ] { assert_eq!(serde_json::from_str::(serialized).unwrap(), vdaf); } diff --git a/tests/integration/new_task.rs b/tests/integration/new_task.rs index 94aabaaf..3f2b1ca0 100644 --- a/tests/integration/new_task.rs +++ b/tests/integration/new_task.rs @@ -30,6 +30,20 @@ pub async fn assert_no_errors(app: &DivviupApi, new_task: &mut NewTask, field: & assert!(errors.is_empty(), "{:?}", errors); } +pub async fn assert_errors_nested( + app: &DivviupApi, + account: Account, + new_task: &mut NewTask, + expected_errors: Value, +) { + let errors = new_task + .normalize_and_validate(account, app.db()) + .await + .unwrap_err(); + let serialized = serde_json::to_value(errors).unwrap(); + assert_eq!(serialized, expected_errors); +} + #[test(harness = set_up)] async fn batch_size(app: DivviupApi) -> TestResult { assert_errors( @@ -139,6 +153,233 @@ async fn time_bucketed_fixed_size(app: DivviupApi) -> TestResult { Ok(()) } +#[test(harness = set_up)] +async fn pure_dp_discrete_laplace(app: DivviupApi) -> TestResult { + let account = fixtures::account(&app).await; + let collector_credential = fixtures::collector_credential(&app, &account).await; + + let mut leader = fixtures::aggregator(&app, None).await.into_active_model(); + leader.role = ActiveValue::Set(Role::Leader); + leader.features = + ActiveValue::Set(Features::from_iter([Feature::PureDpDiscreteLaplace]).into()); + let leader = leader.update(app.db()).await?; + + let mut helper = fixtures::aggregator(&app, None).await.into_active_model(); + helper.role = ActiveValue::Set(Role::Helper); + helper.features = + ActiveValue::Set(Features::from_iter([Feature::PureDpDiscreteLaplace]).into()); + let helper = helper.update(app.db()).await?; + + assert_errors_nested( + &app, + account.clone(), + &mut NewTask { + name: Some("test".into()), + leader_aggregator_id: Some(leader.id.to_string()), + helper_aggregator_id: Some(helper.id.to_string()), + time_precision_seconds: Some(300), + min_batch_size: Some(100), + collector_credential_id: Some(collector_credential.id.to_string()), + vdaf: Some(task::vdaf::Vdaf::Histogram(task::vdaf::Histogram::Opaque( + task::vdaf::BucketLength { + length: 10, + chunk_length: Some(3), + dp_strategy: task::vdaf::DpStrategy { + dp_strategy: task::vdaf::DpStrategyKind::NoDifferentialPrivacy, + budget: task::vdaf::DpBudget { + epsilon: Some(Vec::from([Vec::from([1]), Vec::from([1])])), + }, + }, + }, + ))), + ..Default::default() + }, + json!({ + "vdaf": { + "dp_strategy": [{ + "code": "extra_epsilon", + "message": null, + "params": { + "value": { + "dp_strategy": "NoDifferentialPrivacy", + "budget": { + "epsilon": [[1], [1]], + }, + }, + }, + }], + }, + }), + ) + .await; + assert_errors_nested( + &app, + account.clone(), + &mut NewTask { + name: Some("test".into()), + leader_aggregator_id: Some(leader.id.to_string()), + helper_aggregator_id: Some(helper.id.to_string()), + time_precision_seconds: Some(300), + min_batch_size: Some(100), + collector_credential_id: Some(collector_credential.id.to_string()), + vdaf: Some(task::vdaf::Vdaf::Histogram(task::vdaf::Histogram::Opaque( + task::vdaf::BucketLength { + length: 10, + chunk_length: Some(3), + dp_strategy: task::vdaf::DpStrategy { + dp_strategy: task::vdaf::DpStrategyKind::PureDpDiscreteLaplace, + budget: task::vdaf::DpBudget { epsilon: None }, + }, + }, + ))), + ..Default::default() + }, + json!({ + "vdaf": { + "dp_strategy": [{ + "code": "missing_epsilon", + "message": null, + "params": { + "value": { + "dp_strategy": "PureDpDiscreteLaplace", + "budget": { + "epsilon": null, + }, + }, + }, + }], + }, + }), + ) + .await; + assert_errors_nested( + &app, + account.clone(), + &mut NewTask { + name: Some("test".into()), + leader_aggregator_id: Some(leader.id.to_string()), + helper_aggregator_id: Some(helper.id.to_string()), + time_precision_seconds: Some(300), + min_batch_size: Some(100), + collector_credential_id: Some(collector_credential.id.to_string()), + vdaf: Some(task::vdaf::Vdaf::Histogram(task::vdaf::Histogram::Opaque( + task::vdaf::BucketLength { + length: 10, + chunk_length: Some(3), + dp_strategy: task::vdaf::DpStrategy { + dp_strategy: task::vdaf::DpStrategyKind::PureDpDiscreteLaplace, + budget: task::vdaf::DpBudget { + epsilon: Some(Vec::from([Vec::from([u32::MAX, u32::MAX])])), + }, + }, + }, + ))), + ..Default::default() + }, + json!({ + "vdaf": { + "dp_strategy": { + "budget": { + "epsilon": [{ + "code": "length", + "message": null, + "params": { + "equal": 2, + "value": [[u32::MAX, u32::MAX]], + }, + }], + }, + }, + }, + }), + ) + .await; + + assert_no_errors( + &app, + &mut NewTask { + leader_aggregator_id: Some(leader.id.to_string()), + helper_aggregator_id: Some(helper.id.to_string()), + time_precision_seconds: Some(300), + vdaf: Some(task::vdaf::Vdaf::Histogram(task::vdaf::Histogram::Opaque( + task::vdaf::BucketLength { + length: 10, + chunk_length: Some(3), + dp_strategy: task::vdaf::DpStrategy { + dp_strategy: task::vdaf::DpStrategyKind::PureDpDiscreteLaplace, + budget: task::vdaf::DpBudget { + epsilon: Some(Vec::from([Vec::from([1]), Vec::from([1])])), + }, + }, + }, + ))), + ..Default::default() + }, + "leader_aggregator_id", + ) + .await; + + let mut plain_leader = fixtures::aggregator(&app, None).await.into_active_model(); + plain_leader.role = ActiveValue::Set(Role::Leader); + let plain_leader = plain_leader.update(app.db()).await?; + + let mut plain_helper = fixtures::aggregator(&app, None).await.into_active_model(); + plain_helper.role = ActiveValue::Set(Role::Helper); + let plain_helper = plain_helper.update(app.db()).await?; + + assert_errors( + &app, + &mut NewTask { + leader_aggregator_id: Some(plain_leader.id.to_string()), + helper_aggregator_id: Some(helper.id.to_string()), + time_precision_seconds: Some(300), + vdaf: Some(task::vdaf::Vdaf::Histogram(task::vdaf::Histogram::Opaque( + task::vdaf::BucketLength { + length: 10, + chunk_length: Some(3), + dp_strategy: task::vdaf::DpStrategy { + dp_strategy: task::vdaf::DpStrategyKind::PureDpDiscreteLaplace, + budget: task::vdaf::DpBudget { + epsilon: Some(Vec::from([Vec::from([1]), Vec::from([1])])), + }, + }, + }, + ))), + ..Default::default() + }, + "leader_aggregator_id", + &["pure-dp-discrete-laplace-unsupported"], + ) + .await; + + assert_errors( + &app, + &mut NewTask { + leader_aggregator_id: Some(leader.id.to_string()), + helper_aggregator_id: Some(plain_helper.id.to_string()), + time_precision_seconds: Some(300), + vdaf: Some(task::vdaf::Vdaf::Histogram(task::vdaf::Histogram::Opaque( + task::vdaf::BucketLength { + length: 10, + chunk_length: Some(3), + dp_strategy: task::vdaf::DpStrategy { + dp_strategy: task::vdaf::DpStrategyKind::PureDpDiscreteLaplace, + budget: task::vdaf::DpBudget { + epsilon: Some(Vec::from([Vec::from([1]), Vec::from([1])])), + }, + }, + }, + ))), + ..Default::default() + }, + "helper_aggregator_id", + &["pure-dp-discrete-laplace-unsupported"], + ) + .await; + + Ok(()) +} + #[test(harness = set_up)] async fn aggregator_roles(app: DivviupApi) -> TestResult { let mut leader = fixtures::aggregator(&app, None).await.into_active_model(); diff --git a/tests/integration/vdaf.rs b/tests/integration/vdaf.rs index a9ab83bd..baacc36f 100644 --- a/tests/integration/vdaf.rs +++ b/tests/integration/vdaf.rs @@ -1,4 +1,5 @@ use divviup_api::entity::task::vdaf::{BucketLength, CategoricalBuckets, Histogram, Vdaf}; +use task::vdaf::DpStrategy; use test_support::{assert_eq, test, *}; #[test] pub fn histogram_representations() { @@ -6,17 +7,75 @@ pub fn histogram_representations() { ( json!({"type": "histogram", "buckets": ["a", "b", "c"], "chunk_length": 1}), Protocol::Dap09, - Ok(json!({"Prio3Histogram": {"length": 3, "chunk_length": 1}})), + Ok( + json!({"Prio3Histogram": {"length": 3, "chunk_length": 1, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}}), + ), ), ( json!({"type": "histogram", "buckets": [1, 2, 3], "chunk_length": 2}), Protocol::Dap09, - Ok(json!({"Prio3Histogram": {"length": 4, "chunk_length": 2}})), + Ok( + json!({"Prio3Histogram": {"length": 4, "chunk_length": 2, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}}), + ), ), ( json!({"type": "histogram", "length": 3, "chunk_length": 1}), Protocol::Dap09, - Ok(json!({"Prio3Histogram": {"length": 3, "chunk_length": 1}})), + Ok( + json!({"Prio3Histogram": {"length": 3, "chunk_length": 1, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}}), + ), + ), + ( + json!({"type": "histogram", "length": 3, "chunk_length": 1, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}), + Protocol::Dap09, + Ok( + json!({"Prio3Histogram": {"length": 3, "chunk_length": 1, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}}), + ), + ), + ( + json!({"type": "histogram", "length": 3, "chunk_length": 1, "dp_strategy": {"dp_strategy": "PureDpDiscreteLaplace", "budget": {"epsilon": [[1], [1]]}}}), + Protocol::Dap09, + Ok( + json!({"Prio3Histogram": {"length": 3, "chunk_length": 1, "dp_strategy": {"dp_strategy": "PureDpDiscreteLaplace", "budget": {"epsilon": [[1], [1]]}}}}), + ), + ), + ]; + + for (input, protocol, output) in scenarios { + let vdaf: Vdaf = serde_json::from_value(input.clone()).unwrap(); + assert_eq!( + output, + vdaf.representation_for_protocol(&protocol) + .map(|o| serde_json::to_value(o).unwrap()) + .map_err(|e| serde_json::to_value(e).unwrap()), + "{vdaf:?} {input} {protocol}" + ); + } +} + +#[test] +fn sumvec_representations() { + let scenarios = [ + ( + json!({"type": "sum_vec", "length": 3, "bits": 1, "chunk_length": 1}), + Protocol::Dap09, + Ok( + json!({"Prio3SumVec": {"length": 3, "bits": 1, "chunk_length": 1, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}}), + ), + ), + ( + json!({"type": "sum_vec", "length": 3, "bits": 1, "chunk_length": 1, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}), + Protocol::Dap09, + Ok( + json!({"Prio3SumVec": {"length": 3, "bits": 1, "chunk_length": 1, "dp_strategy": {"dp_strategy": "NoDifferentialPrivacy"}}}), + ), + ), + ( + json!({"type": "sum_vec", "length": 3, "bits": 1, "chunk_length": 1, "dp_strategy": {"dp_strategy": "PureDpDiscreteLaplace", "budget": {"epsilon": [[1], [1]]}}}), + Protocol::Dap09, + Ok( + json!({"Prio3SumVec": {"length": 3, "bits": 1, "chunk_length": 1, "dp_strategy": {"dp_strategy": "PureDpDiscreteLaplace", "budget": {"epsilon": [[1], [1]]}}}}), + ), ), ]; @@ -38,6 +97,7 @@ fn histogram_representation_dap_09_no_chunk_length_1() { let _ = Vdaf::Histogram(Histogram::Categorical(CategoricalBuckets { buckets: Some(Vec::from(["a".to_owned(), "b".to_owned(), "c".to_owned()])), chunk_length: None, + dp_strategy: DpStrategy::default(), })) .representation_for_protocol(&Protocol::Dap09); } @@ -48,6 +108,7 @@ fn histogram_representation_dap_09_no_chunk_length_2() { let _ = Vdaf::Histogram(Histogram::Opaque(BucketLength { length: 3, chunk_length: None, + dp_strategy: DpStrategy::default(), })) .representation_for_protocol(&Protocol::Dap09); }