From e4cf35c38e881aeae3c66da450a4f631159df034 Mon Sep 17 00:00:00 2001 From: Christopher Patton Date: Mon, 5 Aug 2024 17:16:24 -0700 Subject: [PATCH] Resolve open issues for #255 Two questions remain. First, for consistency with other sections, allow the IDPF to define its own type for the public share and express the encoding correction words for {{BBCGG21}} in TLS-syntax. Second, for consistency with other sections, we express the encoding of `Poplar1`'s aggregation parameter in TLS-syntax. This is slightly hairy because of the prefix packing procedure, but this can be factored out pretty nicely. --- draft-irtf-cfrg-vdaf.md | 253 ++++++++++++++++---------------- poc/gen_test_vec.py | 2 +- poc/tests/test_idpf_bbcggi21.py | 18 ++- poc/vdaf_poc/idpf.py | 7 +- poc/vdaf_poc/idpf_bbcggi21.py | 83 ++++++----- poc/vdaf_poc/vdaf_poplar1.py | 52 ++++--- 6 files changed, 225 insertions(+), 190 deletions(-) diff --git a/draft-irtf-cfrg-vdaf.md b/draft-irtf-cfrg-vdaf.md index 0534ea0c..fcd8eae5 100644 --- a/draft-irtf-cfrg-vdaf.md +++ b/draft-irtf-cfrg-vdaf.md @@ -4185,14 +4185,14 @@ denotes either a vector of inner node field elements or leaf node field elements.) The scheme is comprised of the following algorithms: * `idpf.gen(alpha: int, beta_inner: list[list[FieldInner]], beta_leaf: - list[FieldLeaf], nonce: bytes, rand: bytes) -> tuple[bytes, - list[bytes]]` is the randomized IDPF-key generation algorithm. Its inputs are the index `alpha` - the values `beta`, and a nonce string. + list[FieldLeaf], nonce: bytes, rand: bytes) -> tuple[PublicShare, + list[bytes]]` is the randomized IDPF-key generation algorithm. Its inputs are + the index `alpha` the values `beta`, and a nonce string. - The output is a public part that is sent to all Aggregators - and a vector of private IDPF keys, one for each aggregator. The binder string - is used to derive the key in the underlying XofFixedKeyAes128 XOF that is used - for expanding seeds at each level. + The output is a public part (of type `PublicShare`) that is sent to all + Aggregators and a vector of private IDPF keys, one for each aggregator. The + binder string is used to derive the key in the underlying XofFixedKeyAes128 + XOF that is used for expanding seeds at each level. Pre-conditions: @@ -4202,20 +4202,16 @@ elements.) The scheme is comprised of the following algorithms: `range(BITS - 1)`. * `beta_leaf` MUST have length `VALUE_LEN`. * `rand` MUST be generated by a CSPRNG and have length `RAND_SIZE`. - * `nonce` MUST be of length `Idpf.NONCE_SIZE` and chosen uniformly at random by the Client (see - {{nonce-requirements}}). + * `nonce` MUST be of length `Idpf.NONCE_SIZE` and chosen uniformly at + random by the Client (see {{nonce-requirements}}). - > TODO(issue #255) Decide whether to treat the public share as an opaque byte - > string or to replace it with an explicit type. - -* `idpf.eval(agg_id: int, public_share: bytes, key: bytes, level: - int, prefixes: tuple[int, ...], nonce: bytes) -> Output` is the - deterministic, stateless IDPF-key evaluation algorithm run by each - Aggregator. Its inputs are the Aggregator's unique identifier, the public - share distributed to all of the Aggregators, the Aggregator's IDPF key, the - "level" at which to evaluate the IDPF, the sequence of candidate prefixes, - and a nonce string. It returns the share of the value corresponding to each - candidate prefix. +* `idpf.eval(agg_id: int, public_share: PublicShare, key: bytes, level: int, + prefixes: tuple[int, ...], nonce: bytes) -> Output` is the deterministic, + stateless IDPF-key evaluation algorithm run by each Aggregator. Its inputs + are the Aggregator's unique identifier, the public share distributed to all + of the Aggregators, the Aggregator's IDPF key, the "level" at which to + evaluate the IDPF, the sequence of candidate prefixes, and a nonce string. It + returns the share of the value corresponding to each candidate prefix. The output type (i.e., `Output`) depends on the value of `level`: If `level < BITS-1`, the output is the value for an inner node, which has type @@ -4247,18 +4243,19 @@ not include shared state across across VDAF evaluations. In practice, of course, it will often be beneficial to expose a stateful API for IDPFs and carry the state across evaluations. See {{idpf-bbcggi21}} for details. -| Parameter | Description | -|:-----------|:--------------------------| -| SHARES | Number of IDPF keys output by IDPF-key generator | -| BITS | Length in bits of each input string | -| VALUE_LEN | Number of field elements of each output value | -| RAND_SIZE | Size of the random string consumed by the IDPF-key generator. Equal to twice the XOF's seed size. | +| Parameter | Description | +|:------------|:--------------------------| +| SHARES | Number of IDPF keys output by IDPF-key generator | +| BITS | Length in bits of each input string | +| VALUE_LEN | Number of field elements of each output value | +| RAND_SIZE | Size of the random string consumed by the IDPF-key generator. Equal to twice the XOF's seed size. | | NONCE_SIZE | Size of the randon nonce generated by the Client. | -| KEY_SIZE | Size in bytes of each IDPF key | -| FieldInner | Implementation of `Field` ({{field}}) used for values of inner nodes | -| FieldLeaf | Implementation of `Field` used for values of leaf nodes | -| Output | Alias of `list[list[FieldInner]] | list[list[FieldLeaf]]` | -| FieldVec | Alias of `list[FieldInner] | list[FieldLeaf]` | +| KEY_SIZE | Size in bytes of each IDPF key | +| FieldInner | Implementation of `Field` ({{field}}) used for values of inner nodes | +| FieldLeaf | Implementation of `Field` used for values of leaf nodes | +| PublicShare | Type of public share for this IDPF | +| Output | Alias of `list[list[FieldInner]] | list[list[FieldLeaf]]` | +| FieldVec | Alias of `list[FieldInner] | list[FieldLeaf]` | {: #idpf-param title="Constants and types defined by a concrete IDPF."} ### Encoding inputs as indices {#poplar1-idpf-index-encoding} @@ -4302,7 +4299,7 @@ subsections. These methods make use of constants defined in {{poplar1-const}}. | `SHARES` | `2` | | `Measurement` | `int` | | `AggParam` | `tuple[int, Sequence[int]]` | -| `PublicShare` | `bytes` (IDPF public share) | +| `PublicShare` | same as the IDPF | | `InputShare` | `tuple[bytes, bytes, list[FieldInner], list[FieldLeaf]]` | | `OutShare` | `FieldVec` | | `AggShare` | `FieldVec` | @@ -4346,7 +4343,8 @@ def shard( self, measurement: int, nonce: bytes, - rand: bytes) -> tuple[bytes, list[Poplar1InputShare]]: + rand: bytes, + ) -> tuple[Poplar1PublicShare, list[Poplar1InputShare]]: if len(nonce) != self.NONCE_SIZE: raise ValueError("incorrect nonce size") if len(rand) != self.RAND_SIZE: @@ -4481,7 +4479,7 @@ def prep_init( agg_id: int, agg_param: Poplar1AggParam, nonce: bytes, - public_share: bytes, + public_share: Poplar1PublicShare, input_share: Poplar1InputShare) -> tuple[ Poplar1PrepState, FieldVec]: @@ -4745,8 +4743,69 @@ opaque Poplar1FieldLeaf[Fl]; #### Public Share -The public share is equal to the IDPF public share, which is a byte string. -(See {{idpf}}.) +The public share of the IDPF scheme in {{idpf-bbcggi21}} consists of a sequence +of "correction words". A correction word has three components: + +1. the XOF seed of type `bytes`; +2. the control bits of type `tuple[Field2, Field2]`; and +3. the payload of type `list[Field64]` for the first `BITS-1` words and + `list[Field255]` for the last word. + +The encoding is straightforward, except that the control bits are packed as +tightly as possible. The encoded public share is structured as follows: + +~~~ tls-presentation +struct { + Poplar1Seed seed; + Poplar1FieldInner payload[Fi * Poplar1.Idpf.VALUE_LEN]; +} Poplar1CWSeedAndPayloadInner; + +struct { + Poplar1Seed seed; + Poplar1FieldLeaf payload[Fl * Poplar1.Idpf.VALUE_LEN]; +} Poplar1CWSeedAndPayloadLeaf; + +struct { + opaque packed_control_bits[packed_len]; + Poplar1CWSeedAndPayloadInner inner[Ci * (Poplar1.Idpf.BITS-1)]; + Poplar1CWSeedAndPayloadLeaf leaf; +} Poplar1PublicShare; +~~~ + +Here `Ci` denotes the length of `Poplar1ControlWordSeedAndPayloadInner` and +`packed_len = (2*Poplar1.Idpf.BITS + 7) // 8` is the length of the packed +control bits. + +Field `packed_control_bits` is encoded with the following function: + +~~~ python +packed_control_buf = [int(0)] * packed_len +for i, bit in enumerate(control_bits): + packed_control_buf[i // 8] |= bit.as_unsigned() << (i % 8) +packed_control_bits = bytes(packed_control_bits) +~~~ + +Each group of eight bits into a byte, in LSB to MSB order, padding the most +significant bits of the last byte with zeros as necessary, and returns the byte +array. Decoding performs the reverse operation: it takes in a byte array +and a number of bits, and returns a list of bits, extracting eight bits from +each byte in turn, in LSB to MSB order, and stopping after the requested number +of bits. If the byte array has an incorrect length, or if unused bits in the +last bytes are not zero, it throws an error: + +~~~ python +control_bits = [] + for i in range(length): + control_bits.append(Field2( + (packed_control_bits[i // 8] >> (i % 8)) & 1 + )) + leftover_bits = packed_control_bits[-1] >> ( + (length + 7) % 8 + 1 + ) + if (length + 7) // 8 != len(packed_control_bits) or \ + leftover_bits != 0: + raise ValueError('trailing bits') +~~~ #### Input Share @@ -4854,41 +4913,35 @@ struct { The aggregation parameter is encoded as follows: -> TODO(issue #255) Express the aggregation parameter encoding in TLS syntax. -> Decide whether to RECOMMEND this encoding, and if so, add it to test vectors. +~~~ tls-presentation +struct { + uint16_t level; + uint32_t num_prefixes; + opaque packed_prefixes[packed_len]; +} Poplar1AggParam; +~~~ + +The fields in this struct are: `level`, the level of the IDPF tree of each +prefixes; `num_prefixes`, the number of prefixes to evaluate; and +`packed_prefixes`, the sequence of prefixes packed into a byte string of +length `packed_len`. The prefixes are encoded with the following procedure: ~~~ python -def encode_agg_param(self, agg_param: Poplar1AggParam) -> bytes: - level, prefixes = agg_param - if level not in range(2 ** 16): - raise ValueError('level out of range') - if len(prefixes) not in range(2 ** 32): - raise ValueError('number of prefixes out of range') - encoded = bytes() - encoded += to_be_bytes(level, 2) - encoded += to_be_bytes(len(prefixes), 4) - packed = 0 - for (i, prefix) in enumerate(prefixes): - packed |= prefix << ((level + 1) * i) - l = ((level + 1) * len(prefixes) + 7) // 8 - encoded += to_be_bytes(packed, l) - return encoded +packed = 0 +for (i, prefix) in enumerate(prefixes): + packed |= prefix << ((level + 1) * i) +packed_len = ((level + 1) * len(prefixes) + 7) // 8 +packed_prefixes = to_be_bytes(packed, packed_len) +~~~ -def decode_agg_param(self, encoded: bytes) -> Poplar1AggParam: - encoded_level, encoded = encoded[:2], encoded[2:] - level = from_be_bytes(encoded_level) - encoded_prefix_count, encoded = encoded[:4], encoded[4:] - prefix_count = from_be_bytes(encoded_prefix_count) - l = ((level + 1) * prefix_count + 7) // 8 - encoded_packed, encoded = encoded[:l], encoded[l:] - packed = from_be_bytes(encoded_packed) - prefixes = [] - m = 2 ** (level + 1) - 1 - for i in range(prefix_count): - prefixes.append(packed >> ((level + 1) * i) & m) - if len(encoded) != 0: - raise ValueError('trailing bytes') - return (level, tuple(prefixes)) +Decoding involves the following procedure: + +~~~ python +packed = from_be_bytes(packed_prefixes) +prefixes = [] +m = 2 ** (level + 1) - 1 +for i in range(num_prefixes): + prefixes.append(packed >> ((level + 1) * i) & m) ~~~ Implementation note: The aggregation parameter includes the level of the IDPF @@ -4939,13 +4992,15 @@ def gen( beta_inner: list[list[Field64]], beta_leaf: list[Field255], nonce: bytes, - rand: bytes) -> tuple[bytes, list[bytes]]: + rand: bytes) -> tuple[list[CorrectionWord], list[bytes]]: if alpha not in range(2 ** self.BITS): raise ValueError("alpha out of range") if len(beta_inner) != self.BITS - 1: raise ValueError("incorrect beta_inner length") if len(rand) != self.RAND_SIZE: raise ValueError("incorrect rand size") + if len(nonce) != self.NONCE_SIZE: + raise ValueError("incorrect nonce size") key = [ rand[:XofFixedKeyAes128.SEED_SIZE], @@ -4954,7 +5009,7 @@ def gen( seed = key.copy() ctrl = [Field2(0), Field2(1)] - correction_words = [] + public_share = [] for level in range(self.BITS): field: type[Field] field = cast(type[Field], self.current_field(level)) @@ -4994,9 +5049,7 @@ def gen( for i in range(len(w_cw)): w_cw[i] *= mask - correction_words.append((seed_cw, ctrl_cw, w_cw)) - - public_share = self.encode_public_share(correction_words) + public_share.append((seed_cw, ctrl_cw, w_cw)) return (public_share, key) ~~~ {: #idpf-bbcggi21-gen title="IDPF-key generation algorithm of BBCGGI21."} @@ -5013,7 +5066,7 @@ functions `extend()`, `convert()`, and `decode_public_share()` defined in def eval( self, agg_id: int, - public_share: bytes, + public_share: list[CorrectionWord], key: bytes, level: int, prefixes: Sequence[int], @@ -5026,7 +5079,6 @@ def eval( if len(set(prefixes)) != len(prefixes): raise ValueError('prefixes must be unique') - correction_words = self.decode_public_share(public_share) out_share = [] for prefix in prefixes: if prefix not in range(2 ** (level + 1)): @@ -5060,7 +5112,7 @@ def eval( (seed, ctrl, y) = self.eval_next( seed, ctrl, - correction_words[current_level], + public_share[current_level], current_level, bit, nonce, @@ -5078,7 +5130,7 @@ def eval_next( self, prev_seed: bytes, prev_ctrl: Field2, - correction_word: CorrectionWordTuple, + correction_word: CorrectionWord, level: int, bit: int, nonce: bytes) -> tuple[bytes, Field2, FieldVec]: @@ -5143,56 +5195,9 @@ def convert( field = self.current_field(level) w = xof.next_vec(field, self.VALUE_LEN) return (next_seed, cast(FieldVec, w)) - -def encode_public_share( - self, - correction_words: list[CorrectionWordTuple]) -> bytes: - encoded = bytes() - control_bits = list(itertools.chain.from_iterable( - cw[1] for cw in correction_words - )) - encoded += pack_bits(control_bits) - for (level, (seed_cw, _, w_cw)) \ - in enumerate(correction_words): - field = cast(type[Field], self.current_field(level)) - encoded += seed_cw - encoded += field.encode_vec(cast(list[Field], w_cw)) - return encoded - -def decode_public_share( - self, - encoded: bytes) -> list[CorrectionWordTuple]: - l = (2 * self.BITS + 7) // 8 - encoded_ctrl, encoded = encoded[:l], encoded[l:] - control_bits = unpack_bits(encoded_ctrl, 2 * self.BITS) - correction_words = [] - for level in range(self.BITS): - field = self.current_field(level) - ctrl_cw = ( - control_bits[level * 2], - control_bits[level * 2 + 1], - ) - l = XofFixedKeyAes128.SEED_SIZE - seed_cw, encoded = encoded[:l], encoded[l:] - l = field.ENCODED_SIZE * self.VALUE_LEN - encoded_w_cw, encoded = encoded[:l], encoded[l:] - w_cw = field.decode_vec(encoded_w_cw) - correction_words.append((seed_cw, ctrl_cw, w_cw)) - if len(encoded) != 0: - raise ValueError('trailing bytes') - return correction_words ~~~ {: #idpf-bbcggi21-helpers title="Helper functions for the IDPF."} -Here, `pack_bits()` takes a list of bits, packs each group of eight bits into a -byte, in LSB to MSB order, padding the most significant bits of the last byte -with zeros as necessary, and returns the byte array. `unpack_bits()` performs -the reverse operation: it takes in a byte array and a number of bits, and -returns a list of bits, extracting eight bits from each byte in turn, in LSB to -MSB order, and stopping after the requested number of bits. If the byte array -has an incorrect length, or if unused bits in the last bytes are not zero, it -throws an error. - ## Instantiation {#poplar1-inst} By default, Poplar1 is instantiated with the IDPF in {{idpf-bbcggi21}} (`VALUE_LEN diff --git a/poc/gen_test_vec.py b/poc/gen_test_vec.py index 781884fa..a7c7c7bf 100644 --- a/poc/gen_test_vec.py +++ b/poc/gen_test_vec.py @@ -283,7 +283,7 @@ def gen_test_vec_for_idpf(idpf: Idpf, 'beta_inner': printable_beta_inner, 'beta_leaf': printable_beta_leaf, 'nonce': nonce.hex(), - 'public_share': public_share.hex(), + 'public_share': idpf.encode_public_share(public_share).hex(), 'keys': printable_keys, } diff --git a/poc/tests/test_idpf_bbcggi21.py b/poc/tests/test_idpf_bbcggi21.py index 20aa7910..2ef795e8 100644 --- a/poc/tests/test_idpf_bbcggi21.py +++ b/poc/tests/test_idpf_bbcggi21.py @@ -5,7 +5,7 @@ from vdaf_poc.common import from_be_bytes, gen_rand, vec_add from vdaf_poc.field import Field from vdaf_poc.idpf import Idpf -from vdaf_poc.idpf_bbcggi21 import IdpfBBCGGI21 +from vdaf_poc.idpf_bbcggi21 import IdpfBBCGGI21, CorrectionWord class TestIdpfBBCGGI21(unittest.TestCase): @@ -132,7 +132,7 @@ def test_index_encoding(self) -> None: idpf = IdpfBBCGGI21(1, 32) nonce = gen_rand(idpf.NONCE_SIZE) - def shard(s: bytes) -> tuple[bytes, list[bytes]]: + def shard(s: bytes) -> tuple[list[CorrectionWord], list[bytes]]: alpha = from_be_bytes(s) beta_inner = [[idpf.field_inner(1)]] * (idpf.BITS - 1) beta_leaf = [idpf.field_leaf(1)] @@ -167,3 +167,17 @@ def test_is_prefix(self) -> None: self.assertTrue(idpf.is_prefix(0b1100, 0b11000001, 3)) self.assertFalse(idpf.is_prefix(0b111, 0b11000001, 2)) self.assertFalse(idpf.is_prefix(0b1101, 0b11000001, 3)) + + def test_public_share_roundtrip(self) -> None: + idpf = IdpfBBCGGI21(1, 32) + alpha = from_be_bytes(b"cool") + beta_inner = [[idpf.field_inner(23)]] * (idpf.BITS - 1) + beta_leaf = [idpf.field_leaf(97)] + nonce = gen_rand(idpf.NONCE_SIZE) + rand = gen_rand(idpf.RAND_SIZE) + (public_share, _keys) = idpf.gen( + alpha, beta_inner, beta_leaf, nonce, rand) + self.assertEqual( + idpf.decode_public_share(idpf.encode_public_share(public_share)), + public_share, + ) diff --git a/poc/vdaf_poc/idpf.py b/poc/vdaf_poc/idpf.py index f54bf910..b08bca7b 100644 --- a/poc/vdaf_poc/idpf.py +++ b/poc/vdaf_poc/idpf.py @@ -7,6 +7,7 @@ FieldInner = TypeVar("FieldInner", bound=Field) FieldLeaf = TypeVar("FieldLeaf", bound=Field) +PublicShare = TypeVar("PublicShare") # Type alias for the output of `eval()`. Output: TypeAlias = list[list[FieldInner]] | list[list[FieldLeaf]] @@ -14,7 +15,7 @@ FieldVec: TypeAlias = list[FieldInner] | list[FieldLeaf] -class Idpf(Generic[FieldInner, FieldLeaf], metaclass=ABCMeta): +class Idpf(Generic[FieldInner, FieldLeaf, PublicShare], metaclass=ABCMeta): """ An Incremental Distributed Point Function (IDPF). @@ -70,7 +71,7 @@ def gen(self, beta_inner: list[list[FieldInner]], beta_leaf: list[FieldLeaf], nonce: bytes, - rand: bytes) -> tuple[bytes, list[bytes]]: + rand: bytes) -> tuple[PublicShare, list[bytes]]: """ Generates an IDPF public share and sequence of IDPF-keys of length `SHARES`. Input `alpha` is the index to encode. Inputs `beta_inner` and @@ -91,7 +92,7 @@ def gen(self, @abstractmethod def eval(self, agg_id: int, - public_share: bytes, + public_share: PublicShare, key: bytes, level: int, prefixes: Sequence[int], diff --git a/poc/vdaf_poc/idpf_bbcggi21.py b/poc/vdaf_poc/idpf_bbcggi21.py index 8c673119..936b7a8c 100644 --- a/poc/vdaf_poc/idpf_bbcggi21.py +++ b/poc/vdaf_poc/idpf_bbcggi21.py @@ -24,10 +24,9 @@ # types, which aids with self-documentation. FieldVec: TypeAlias = list[Field64] | list[Field255] -CorrectionWordTuple: TypeAlias = tuple[bytes, tuple[Field2, Field2], FieldVec] +CorrectionWord: TypeAlias = tuple[bytes, tuple[Field2, Field2], FieldVec] - -class IdpfBBCGGI21(Idpf[Field64, Field255]): +class IdpfBBCGGI21(Idpf[Field64, Field255, list[CorrectionWord]]): """ The IDPF of {{BBCGGI21}}, Section 6. It is identical except that the output shares may be tuples rather than single field elements. In particular, the @@ -72,7 +71,7 @@ def gen( beta_inner: list[list[Field64]], beta_leaf: list[Field255], nonce: bytes, - rand: bytes) -> tuple[bytes, list[bytes]]: + rand: bytes) -> tuple[list[CorrectionWord], list[bytes]]: if alpha not in range(2 ** self.BITS): raise ValueError("alpha out of range") if len(beta_inner) != self.BITS - 1: @@ -89,7 +88,7 @@ def gen( seed = key.copy() ctrl = [Field2(0), Field2(1)] - correction_words = [] + public_share = [] for level in range(self.BITS): field: type[Field] field = cast(type[Field], self.current_field(level)) @@ -129,9 +128,7 @@ def gen( for i in range(len(w_cw)): w_cw[i] *= mask - correction_words.append((seed_cw, ctrl_cw, w_cw)) - - public_share = self.encode_public_share(correction_words) + public_share.append((seed_cw, ctrl_cw, w_cw)) return (public_share, key) # NOTE: The eval() and eval_next(), and prep_shares_to_prep() methods @@ -143,7 +140,7 @@ def gen( def eval( self, agg_id: int, - public_share: bytes, + public_share: list[CorrectionWord], key: bytes, level: int, prefixes: Sequence[int], @@ -155,7 +152,6 @@ def eval( if len(set(prefixes)) != len(prefixes): raise ValueError('prefixes must be unique') - correction_words = self.decode_public_share(public_share) out_share = [] for prefix in prefixes: if prefix not in range(2 ** (level + 1)): @@ -189,7 +185,7 @@ def eval( (seed, ctrl, y) = self.eval_next( seed, ctrl, - correction_words[current_level], + public_share[current_level], current_level, bit, nonce, @@ -207,7 +203,7 @@ def eval_next( self, prev_seed: bytes, prev_ctrl: Field2, - correction_word: CorrectionWordTuple, + correction_word: CorrectionWord, level: int, bit: int, nonce: bytes) -> tuple[bytes, Field2, FieldVec]: @@ -240,10 +236,9 @@ def eval_next( return (next_seed, next_ctrl, cast(FieldVec, y)) - # NOTE: The extend(), convert(), encode_public_share(), and - # decode_public_share() methods are excerpted in the document, - # de-indented, as figure {{idpf-poplar-helpers}}. Their width should - # be limited to 69 columns after de-indenting, or 73 columns before + # NOTE: The extend() and convert() methods are excerpted in the document, + # de-indented, as figure {{idpf-poplar-helpers}}. Their width should be + # limited to 69 columns after de-indenting, or 73 columns before # de-indenting, to avoid warnings from xml2rfc. # =================================================================== def extend( @@ -276,14 +271,14 @@ def convert( def encode_public_share( self, - correction_words: list[CorrectionWordTuple]) -> bytes: + public_share: list[CorrectionWord]) -> bytes: encoded = bytes() control_bits = list(itertools.chain.from_iterable( - cw[1] for cw in correction_words + cw[1] for cw in public_share )) encoded += pack_bits(control_bits) for (level, (seed_cw, _, w_cw)) \ - in enumerate(correction_words): + in enumerate(public_share): field = cast(type[Field], self.current_field(level)) encoded += seed_cw encoded += field.encode_vec(cast(list[Field], w_cw)) @@ -291,11 +286,11 @@ def encode_public_share( def decode_public_share( self, - encoded: bytes) -> list[CorrectionWordTuple]: + encoded: bytes) -> list[CorrectionWord]: l = (2 * self.BITS + 7) // 8 encoded_ctrl, encoded = encoded[:l], encoded[l:] control_bits = unpack_bits(encoded_ctrl, 2 * self.BITS) - correction_words = [] + public_share = [] for level in range(self.BITS): field = self.current_field(level) ctrl_cw = ( @@ -307,29 +302,41 @@ def decode_public_share( l = field.ENCODED_SIZE * self.VALUE_LEN encoded_w_cw, encoded = encoded[:l], encoded[l:] w_cw = field.decode_vec(encoded_w_cw) - correction_words.append((seed_cw, ctrl_cw, w_cw)) + public_share.append((seed_cw, ctrl_cw, w_cw)) if len(encoded) != 0: raise ValueError('trailing bytes') - return correction_words - + return public_share -def pack_bits(bits: list[Field2]) -> bytes: - byte_len = (len(bits) + 7) // 8 - packed = [int(0)] * byte_len - for i, bit in enumerate(bits): - packed[i // 8] |= bit.as_unsigned() << (i % 8) - return bytes(packed) - -def unpack_bits(packed: bytes, length: int) -> list[Field2]: - bits = [] +def pack_bits(control_bits: list[Field2]) -> bytes: + packed_len = (len(control_bits) + 7) // 8 + # NOTE: The following is excerpted in the document, de-indented. Thee width + # should be limited to 69 columns after de-indenting, or 73 columns before, + # to avoid warnings from xml2rfc. + # =================================================================== + packed_control_buf = [int(0)] * packed_len + for i, bit in enumerate(control_bits): + packed_control_buf[i // 8] |= bit.as_unsigned() << (i % 8) + packed_control_bits = bytes(packed_control_buf) + # NOTE: End of exerpt. + return packed_control_bits + + +def unpack_bits(packed_control_bits: bytes, length: int) -> list[Field2]: + # NOTE: The following is excerpted in the document, de-indented. Thee width + # should be limited to 69 columns after de-indenting, or 73 columns before, + # to avoid warnings from xml2rfc. + # =================================================================== + control_bits = [] for i in range(length): - bits.append(Field2( - (packed[i // 8] >> (i % 8)) & 1 + control_bits.append(Field2( + (packed_control_bits[i // 8] >> (i % 8)) & 1 )) - leftover_bits = packed[-1] >> ( + leftover_bits = packed_control_bits[-1] >> ( (length + 7) % 8 + 1 ) - if (length + 7) // 8 != len(packed) or leftover_bits != 0: + if (length + 7) // 8 != len(packed_control_bits) or \ + leftover_bits != 0: raise ValueError('trailing bits') - return bits + # NOTE: End of exerpt. + return control_bits diff --git a/poc/vdaf_poc/vdaf_poplar1.py b/poc/vdaf_poc/vdaf_poplar1.py index 32c2252a..f9cfb710 100644 --- a/poc/vdaf_poc/vdaf_poplar1.py +++ b/poc/vdaf_poc/vdaf_poplar1.py @@ -1,12 +1,12 @@ """The Poplar1 VDAF.""" -from typing import Any, Optional, Sequence, TypeAlias, cast +from typing import Any, Optional, Sequence, TypeAlias, TypeVar, cast from vdaf_poc.common import (byte, from_be_bytes, front, to_be_bytes, vec_add, vec_sub) from vdaf_poc.field import Field, Field64, Field255 from vdaf_poc.idpf import Idpf -from vdaf_poc.idpf_bbcggi21 import IdpfBBCGGI21 +from vdaf_poc.idpf_bbcggi21 import IdpfBBCGGI21, CorrectionWord from vdaf_poc.vdaf import Vdaf from vdaf_poc.xof import Xof, XofTurboShake128 @@ -20,6 +20,7 @@ int, # level Sequence[int], # prefixes ] +Poplar1PublicShare: TypeAlias = list[CorrectionWord] Poplar1InputShare: TypeAlias = tuple[ bytes, # IDPF key bytes, # correlated randomness seed @@ -37,7 +38,7 @@ class Poplar1( Vdaf[ int, # Measurement, `range(0, 2 ** BITS)` Poplar1AggParam, # AggParam - bytes, # PublicShare, encoded IDPF public share + Poplar1PublicShare, # PublicShare Poplar1InputShare, # InputShare FieldVec, # OutShare FieldVec, # AggShare @@ -47,7 +48,7 @@ class Poplar1( Optional[FieldVec], # PrepMessage ]): - idpf: Idpf[Field64, Field255] + idpf: IdpfBBCGGI21 xof: type[Xof] ID = 0x00001000 @@ -73,7 +74,8 @@ def shard( self, measurement: int, nonce: bytes, - rand: bytes) -> tuple[bytes, list[Poplar1InputShare]]: + rand: bytes, + ) -> tuple[Poplar1PublicShare, list[Poplar1InputShare]]: if len(nonce) != self.NONCE_SIZE: raise ValueError("incorrect nonce size") if len(rand) != self.RAND_SIZE: @@ -227,7 +229,7 @@ def prep_init( agg_id: int, agg_param: Poplar1AggParam, nonce: bytes, - public_share: bytes, + public_share: Poplar1PublicShare, input_share: Poplar1InputShare) -> tuple[ Poplar1PrepState, FieldVec]: @@ -402,11 +404,6 @@ def unshard( agg = vec_add(agg, cast(list[Field], agg_share)) return [x.as_unsigned() for x in agg] - # NOTE: The encode_agg_param() and decode_agg_param() methods are - # excerpted in the document, de-indented. Their width should be - # limited to 69 columns after de-indenting, or 73 columns before - # de-indenting, to avoid warnings from xml2rfc. - # =================================================================== def encode_agg_param(self, agg_param: Poplar1AggParam) -> bytes: level, prefixes = agg_param if level not in range(2 ** 16): @@ -416,25 +413,36 @@ def encode_agg_param(self, agg_param: Poplar1AggParam) -> bytes: encoded = bytes() encoded += to_be_bytes(level, 2) encoded += to_be_bytes(len(prefixes), 4) + # NOTE: The following lines are exerpted in the document. Their width + # should be limited to 69 columns after de-indenting, or 77 columns + # before de-indenting, to avoid warnings from xml2rfc. + # =================================================================== packed = 0 for (i, prefix) in enumerate(prefixes): packed |= prefix << ((level + 1) * i) - l = ((level + 1) * len(prefixes) + 7) // 8 - encoded += to_be_bytes(packed, l) + packed_len = ((level + 1) * len(prefixes) + 7) // 8 + packed_prefixes = to_be_bytes(packed, packed_len) + # NOTE: End of excerpt. + encoded += packed_prefixes return encoded def decode_agg_param(self, encoded: bytes) -> Poplar1AggParam: - encoded_level, encoded = encoded[:2], encoded[2:] + encoded_level, encoded = front(2, encoded) level = from_be_bytes(encoded_level) - encoded_prefix_count, encoded = encoded[:4], encoded[4:] - prefix_count = from_be_bytes(encoded_prefix_count) - l = ((level + 1) * prefix_count + 7) // 8 - encoded_packed, encoded = encoded[:l], encoded[l:] - packed = from_be_bytes(encoded_packed) + encoded_num_prefixes, encoded = front(4, encoded) + num_prefixes = from_be_bytes(encoded_num_prefixes) + packed_len = ((level + 1) * num_prefixes + 7) // 8 + packed_prefixes, encoded = front(packed_len, encoded) + # NOTE: The following lines are exerpted in the document. Their width + # should be limited to 69 columns after de-indenting, or 77 columns + # before de-indenting, to avoid warnings from xml2rfc. + # =================================================================== + packed = from_be_bytes(packed_prefixes) prefixes = [] m = 2 ** (level + 1) - 1 - for i in range(prefix_count): + for i in range(num_prefixes): prefixes.append(packed >> ((level + 1) * i) & m) + # NOTE: End of excerpt. if len(encoded) != 0: raise ValueError('trailing bytes') return (level, tuple(prefixes)) @@ -452,8 +460,8 @@ def test_vec_encode_input_share(self, input_share: Poplar1InputShare) -> bytes: encoded += self.idpf.field_leaf.encode_vec(leaf) return encoded - def test_vec_encode_public_share(self, public_share: bytes) -> bytes: - return public_share + def test_vec_encode_public_share(self, public_share: Poplar1PublicShare) -> bytes: + return self.idpf.encode_public_share(public_share) def test_vec_encode_agg_share(self, agg_share: FieldVec) -> bytes: return encode_idpf_field_vec(agg_share)