From d0828bd81bcefe6f992d6909128b2b7f1c6d5157 Mon Sep 17 00:00:00 2001 From: Martin Vrachev Date: Tue, 25 May 2021 18:09:12 +0300 Subject: [PATCH] New API: Comprehensive serialization testing The idea of this commit is to separate (de)serialization testing outside test_api.py and make sure we are testing from_dict/to_dict for all possible valid data for all classes. Jussi in his comment here: https://github.com/theupdateframework/tuf/issues/1391#issuecomment-849390669 proposed using decorators when creating comprehensive testing for metadata serialization. The main problems he pointed out is that: 1) there is a lot of code needed to generate the data for each case 2) the test implementation scales badly when you want to add new cases for your tests, then you would have to add code as well 3) the dictionary format is not visible - we are loading external files and assuming they are not changed and valid In this change, I am using a decorator with an argument that complicates the implementation of the decorator and requires three nested functions, but the advantages are that we are resolving the above three problems: 1) we don't need new code when adding a new test case 2) a small amount of hardcoded data is required for each new test 3) the dictionaries are all in the test module without the need of creating new directories and copying data. Signed-off-by: Martin Vrachev --- tests/test_api.py | 168 +--------------------- tests/test_metadata_serialization.py | 203 +++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 167 deletions(-) create mode 100644 tests/test_metadata_serialization.py diff --git a/tests/test_api.py b/tests/test_api.py index e0b57114b2..b9530900bf 100755 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -243,49 +243,6 @@ def test_metadata_base(self): Metadata.from_dict(data) - def test_metafile_class(self): - # Test from_dict and to_dict with all attributes. - data = { - "hashes": { - "sha256": "8f88e2ba48b412c3843e9bb26e1b6f8fc9e98aceb0fbaa97ba37b4c98717d7ab" - }, - "length": 515, - "version": 1 - } - metafile_obj = MetaFile.from_dict(copy.copy(data)) - self.assertEqual(metafile_obj.to_dict(), data) - - # Test from_dict and to_dict without length. - del data["length"] - metafile_obj = MetaFile.from_dict(copy.copy(data)) - self.assertEqual(metafile_obj.to_dict(), data) - - # Test from_dict and to_dict without length and hashes. - del data["hashes"] - metafile_obj = MetaFile.from_dict(copy.copy(data)) - self.assertEqual(metafile_obj.to_dict(), data) - - - def test_targetfile_class(self): - # Test from_dict and to_dict with all attributes. - data = { - "custom": { - "file_permissions": "0644" - }, - "hashes": { - "sha256": "65b8c67f51c993d898250f40aa57a317d854900b3a04895464313e48785440da", - "sha512": "467430a68afae8e9f9c0771ea5d78bf0b3a0d79a2d3d3b40c69fde4dd42c461448aef76fcef4f5284931a1ffd0ac096d138ba3a0d6ca83fa8d7285a47a296f77" - }, - "length": 31 - } - targetfile_obj = TargetFile.from_dict(copy.copy(data)) - self.assertEqual(targetfile_obj.to_dict(), data) - - # Test from_dict and to_dict without custom. - del data["custom"] - targetfile_obj = TargetFile.from_dict(copy.copy(data)) - self.assertEqual(targetfile_obj.to_dict(), data) - def test_metadata_snapshot(self): snapshot_path = os.path.join( self.repo_dir, 'metadata', 'snapshot.json') @@ -304,13 +261,6 @@ def test_metadata_snapshot(self): snapshot.signed.meta['role1.json'].to_dict(), fileinfo.to_dict() ) - # Test from_dict and to_dict without hashes and length. - snapshot_dict = snapshot.to_dict() - del snapshot_dict['signed']['meta']['role1.json']['length'] - del snapshot_dict['signed']['meta']['role1.json']['hashes'] - test_dict = copy.deepcopy(snapshot_dict['signed']) - snapshot = Snapshot.from_dict(test_dict) - self.assertEqual(snapshot_dict['signed'], snapshot.to_dict()) def test_metadata_timestamp(self): timestamp_path = os.path.join( @@ -349,13 +299,6 @@ def test_metadata_timestamp(self): timestamp.signed.meta['snapshot.json'].to_dict(), fileinfo.to_dict() ) - # Test from_dict and to_dict without hashes and length. - timestamp_dict = timestamp.to_dict() - del timestamp_dict['signed']['meta']['snapshot.json']['length'] - del timestamp_dict['signed']['meta']['snapshot.json']['hashes'] - test_dict = copy.deepcopy(timestamp_dict['signed']) - timestamp_test = Timestamp.from_dict(test_dict) - self.assertEqual(timestamp_dict['signed'], timestamp_test.to_dict()) def test_key_class(self): @@ -369,21 +312,12 @@ def test_key_class(self): }, } for key_dict in keys.values(): - # Testing that the workflow of deserializing and serializing - # a key dictionary doesn't change the content. - test_key_dict = key_dict.copy() - key_obj = Key.from_dict("id", test_key_dict) - self.assertEqual(key_dict, key_obj.to_dict()) # Test creating an instance without a required attribute. for key in key_dict.keys(): test_key_dict = key_dict.copy() del test_key_dict[key] with self.assertRaises(KeyError): Key.from_dict("id", test_key_dict) - # Test creating a Key instance with wrong keyval format. - key_dict["keyval"] = {} - with self.assertRaises(KeyError): - Key.from_dict("id", key_dict) def test_role_class(self): @@ -402,23 +336,12 @@ def test_role_class(self): }, } for role_dict in roles.values(): - # Testing that the workflow of deserializing and serializing - # a role dictionary doesn't change the content. - test_role_dict = role_dict.copy() - role_obj = Role.from_dict(test_role_dict) - self.assertEqual(role_dict, role_obj.to_dict()) # Test creating an instance without a required attribute. for role_attr in role_dict.keys(): test_role_dict = role_dict.copy() del test_role_dict[role_attr] with self.assertRaises(KeyError): - Key.from_dict("id", test_role_dict) - # Test creating a Role instance with keyid dublicates. - # for keyid in role_dict["keyids"]: - role_dict["keyids"].append(role_dict["keyids"][0]) - test_role_dict = role_dict.copy() - with self.assertRaises(ValueError): - Role.from_dict(test_role_dict) + Role.from_dict(test_role_dict) def test_metadata_root(self): @@ -465,84 +388,8 @@ def test_metadata_root(self): with self.assertRaises(KeyError): root.signed.remove_key('root', 'nosuchkey') - # Test serializing and deserializing without consistent_snapshot. - root_dict = root.to_dict() - del root_dict["signed"]["consistent_snapshot"] - root = Root.from_dict(copy.deepcopy(root_dict["signed"])) - self.assertEqual(root_dict["signed"], root.to_dict()) - - def test_delegated_role_class(self): - roles = [ - { - "keyids": [ - "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a" - ], - "name": "role1", - "paths": [ - "file3.txt" - ], - "terminating": False, - "threshold": 1 - } - ] - for role in roles: - # Testing that the workflow of deserializing and serializing - # a delegation role dictionary doesn't change the content. - key_obj = DelegatedRole.from_dict(role.copy()) - self.assertEqual(role, key_obj.to_dict()) - - # Test creating a DelegatedRole object with both "paths" and - # "path_hash_prefixes" set. - role["path_hash_prefixes"] = "foo" - with self.assertRaises(ValueError): - DelegatedRole.from_dict(role.copy()) - - # Test creating DelegatedRole only with "path_hash_prefixes" (an empty one) - del role["paths"] - role["path_hash_prefixes"] = [] - role_obj = DelegatedRole.from_dict(role.copy()) - self.assertEqual(role_obj.to_dict(), role) - - # Test creating DelegatedRole only with "paths" (now an empty one) - del role["path_hash_prefixes"] - role["paths"] = [] - role_obj = DelegatedRole.from_dict(role.copy()) - self.assertEqual(role_obj.to_dict(), role) - - # Test creating DelegatedRole without "paths" and - # "path_hash_prefixes" set - del role["paths"] - role_obj = DelegatedRole.from_dict(role.copy()) - self.assertEqual(role_obj.to_dict(), role) - def test_delegation_class(self): - roles = [ - { - "keyids": [ - "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a" - ], - "name": "role1", - "paths": [ - "file3.txt" - ], - "terminating": False, - "threshold": 1 - } - ] - keys = { - "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d":{ - "keytype": "ed25519", - "keyval": { - "public": "edcd0a32a07dce33f7c7873aaffbff36d20ea30787574ead335eefd337e4dacd" - }, - "scheme": "ed25519" - }, - } - delegations_dict = {"keys": keys, "roles": roles} - delegations = Delegations.from_dict(copy.deepcopy(delegations_dict)) - self.assertEqual(delegations_dict, delegations.to_dict()) - # empty keys and roles delegations_dict = {"keys":{}, "roles":[]} delegations = Delegations.from_dict(delegations_dict.copy()) @@ -584,19 +431,6 @@ def test_metadata_targets(self): targets.signed.targets[filename].to_dict(), fileinfo.to_dict() ) - # Test from_dict/to_dict Targets with empty targets. - targets_dict = copy.deepcopy(targets.to_dict()) - targets_dict["signed"]["targets"] = {} - tmp_dict = copy.deepcopy(targets_dict["signed"]) - targets_obj = Targets.from_dict(tmp_dict) - self.assertEqual(targets_dict["signed"], targets_obj.to_dict()) - - # Test from_dict/to_dict Targets without delegations - targets_dict = targets.to_dict() - del targets_dict["signed"]["delegations"] - tmp_dict = copy.deepcopy(targets_dict["signed"]) - targets_obj = Targets.from_dict(tmp_dict) - self.assertEqual(targets_dict["signed"], targets_obj.to_dict()) def setup_dict_with_unrecognized_field(self, file_path, field, value): json_dict = {} diff --git a/tests/test_metadata_serialization.py b/tests/test_metadata_serialization.py new file mode 100644 index 0000000000..920899599b --- /dev/null +++ b/tests/test_metadata_serialization.py @@ -0,0 +1,203 @@ +# Copyright New York University and the TUF contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +""" Unit tests testing tuf/api/metadata.py classes +serialization and deserialization. + +""" + +import json +import sys +import logging +import unittest +import copy + +from typing import Dict, Callable + +from tests import utils + +from tuf.api.metadata import ( + Root, + Snapshot, + Timestamp, + Targets, + Key, + Role, + MetaFile, + TargetFile, + Delegations, + DelegatedRole, +) + +logger = logging.getLogger(__name__) + +# DataSet is only here so type hints can be used: +# It is a dict of name to test dict +DataSet = Dict[str, str] + +# Test runner decorator: Runs the test as a set of N SubTests, +# (where N is number of items in dataset), feeding the actual test +# function one test case at a time +def run_sub_tests_with_dataset(dataset: DataSet): + def real_decorator(function: Callable[["TestSerialization", str], None]): + def wrapper(test_cls: "TestSerialization"): + for case, data in dataset.items(): + with test_cls.subTest(case=case): + function(test_cls, data) + return wrapper + return real_decorator + + +class TestSerialization(unittest.TestCase): + + valid_keys: DataSet = { + "all": '{"keytype": "rsa", "scheme": "rsassa-pss-sha256", \ + "keyval": {"public": "foo"}}', + } + + @run_sub_tests_with_dataset(valid_keys) + def test_key_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + key = Key.from_dict("id", copy.copy(case_dict)) + self.assertDictEqual(case_dict, key.to_dict()) + + + valid_roles: DataSet = { + "all": '{"keyids": ["keyid"], "threshold": 3}' + } + + @run_sub_tests_with_dataset(valid_roles) + def test_role_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + role = Role.from_dict(copy.deepcopy(case_dict)) + self.assertDictEqual(case_dict, role.to_dict()) + + + valid_roots: DataSet = { + "all": '{"_type": "root", "spec_version": "1.0.0", "version": 1, \ + "expires": "2030-01-01T00:00:00Z", "consistent_snapshot": false, \ + "keys": {"keyid" : {"keytype": "rsa", "scheme": "rsassa-pss-sha256", "keyval": {"public": "foo"}}}, \ + "roles": { "targets": {"keyids": ["keyid"], "threshold": 3}} \ + }', + "no consistent_snapshot": '{ "_type": "root", "spec_version": "1.0.0", "version": 1, \ + "expires": "2030-01-01T00:00:00Z", \ + "keys": {"keyid" : {"keytype": "rsa", "scheme": "rsassa-pss-sha256", "keyval": {"public": "foo"} }}, \ + "roles": { "targets": {"keyids": ["keyid"], "threshold": 3} } \ + }', + } + + @run_sub_tests_with_dataset(valid_roots) + def test_root_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + root = Root.from_dict(copy.deepcopy(case_dict)) + self.assertDictEqual(case_dict, root.to_dict()) + + valid_metafiles: DataSet = { + "all": '{"hashes": {"sha256" : "abc"}, "length": 12, "version": 1}', + "no length": '{"hashes": {"sha256" : "abc"}, "version": 1 }', + "no hashes": '{"length": 12, "version": 1}' + } + + @run_sub_tests_with_dataset(valid_metafiles) + def test_metafile_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + metafile = MetaFile.from_dict(copy.copy(case_dict)) + self.assertDictEqual(case_dict, metafile.to_dict()) + + + valid_timestamps: DataSet = { + "all": '{ "_type": "timestamp", "spec_version": "1.0.0", "version": 1, "expires": "2030-01-01T00:00:00Z", \ + "meta": {"snapshot.json": {"hashes": {"sha256" : "abc"}, "version": 1}}}' + } + + @run_sub_tests_with_dataset(valid_timestamps) + def test_timestamp_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + timestamp = Timestamp.from_dict(copy.deepcopy(case_dict)) + self.assertDictEqual(case_dict, timestamp.to_dict()) + + + valid_snapshots: DataSet = { + "all": '{ "_type": "snapshot", "spec_version": "1.0.0", "version": 1, "expires": "2030-01-01T00:00:00Z", \ + "meta": { "file.txt": { "hashes": {"sha256" : "abc"}, "version": 1 }}}' + } + + @run_sub_tests_with_dataset(valid_snapshots) + def test_snapshot_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + snapshot = Snapshot.from_dict(copy.deepcopy(case_dict)) + self.assertDictEqual(case_dict, snapshot.to_dict()) + + + valid_delegated_roles: DataSet = { + "no hash prefix attribute": + '{"keyids": ["keyid"], "name": "a", "paths": ["fn1", "fn2"], \ + "terminating": false, "threshold": 1}', + "no path attribute": + '{"keyids": ["keyid"], "name": "a", "terminating": false, \ + "path_hash_prefixes": ["h1", "h2"], "threshold": 99}', + "no hash or path prefix": + '{"keyids": ["keyid"], "name": "a", "terminating": true, "threshold": 3}', + } + + @run_sub_tests_with_dataset(valid_delegated_roles) + def test_delegated_role_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + deserialized_role = DelegatedRole.from_dict(copy.copy(case_dict)) + self.assertDictEqual(case_dict, deserialized_role.to_dict()) + + + valid_delegations: DataSet = { + "all": '{"keys": {"keyid" : {"keytype": "rsa", "scheme": "rsassa-pss-sha256", "keyval": {"public": "foo"}}}, \ + "roles": [ {"keyids": ["keyid"], "name": "a", "terminating": true, "threshold": 3} ]}' + } + + @run_sub_tests_with_dataset(valid_delegations) + def test_delegation_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + delegation = Delegations.from_dict(copy.deepcopy(case_dict)) + self.assertDictEqual(case_dict, delegation.to_dict()) + + + valid_targetfiles: DataSet = { + "all": '{"length": 12, "hashes": {"sha256" : "abc"}, \ + "custom" : {"foo": "bar"} }', + "no custom": '{"length": 12, "hashes": {"sha256" : "abc"}}' + } + + @run_sub_tests_with_dataset(valid_targetfiles) + def test_targetfile_serialization(self, test_case_data: str): + case_dict = json.loads(test_case_data) + target_file = TargetFile.from_dict(copy.copy(case_dict)) + self.assertDictEqual(case_dict, target_file.to_dict()) + + + valid_targets: DataSet = { + "all attributes": '{"_type": "targets", "spec_version": "1.0.0", "version": 1, "expires": "2030-01-01T00:00:00Z", \ + "targets": { "file.txt": {"length": 12, "hashes": {"sha256" : "abc"} } }, \ + "delegations": {"keys": {"keyid" : {"keytype": "rsa", \ + "scheme": "rsassa-pss-sha256", "keyval": {"public": "foo"} }}, \ + "roles": [ {"keyids": ["keyid"], "name": "a", "terminating": true, "threshold": 3} ]} \ + }', + "empty targets": '{"_type": "targets", "spec_version": "1.0.0", "version": 1, "expires": "2030-01-01T00:00:00Z", \ + "targets": {}, \ + "delegations": {"keys": {"keyid" : {"keytype": "rsa", \ + "scheme": "rsassa-pss-sha256", "keyval": {"public": "foo"} }}, \ + "roles": [ {"keyids": ["keyid"], "name": "a", "terminating": true, "threshold": 3} ]} \ + }', + "no delegations": '{"_type": "targets", "spec_version": "1.0.0", "version": 1, "expires": "2030-01-01T00:00:00Z", \ + "targets": { "file.txt": {"length": 12, "hashes": {"sha256" : "abc"} } } \ + }' + } + + @run_sub_tests_with_dataset(valid_targets) + def test_targets_serialization(self, test_case_data): + case_dict = json.loads(test_case_data) + targets = Targets.from_dict(copy.deepcopy(case_dict)) + self.assertDictEqual(case_dict, targets.to_dict()) + + +# Run unit test. +if __name__ == '__main__': + utils.configure_test_logging(sys.argv) + unittest.main()