From 6f7c5415316ecbea7c3e2ed5f4e4308c4e20c18a Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 3 Aug 2020 10:44:29 -0700 Subject: [PATCH 01/10] This WIP commit creates a separate keydb for each delegating instance. This means that all roles delegated to by root will look for keys from the root metadata, roles delegated to by targets will look for keys in targets metadata, and so on. This commit adds this functionality to the keydb and ensures that these separate keydbs are created and used for storing keys. It requires a future change to the roledb to store the delegating role and further testing. Signed-off-by: marinamoore --- tuf/client/updater.py | 62 ++++++++++---------------- tuf/developer_tool.py | 51 ++++++++++++---------- tuf/keydb.py | 98 +++++++++++++++++++++++++++++++++++++++++- tuf/repository_lib.py | 59 +++++++++++-------------- tuf/repository_tool.py | 24 +++-------- tuf/sig.py | 9 ++-- 6 files changed, 181 insertions(+), 122 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index dd292c26ac..bdd0dc3b61 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -946,36 +946,14 @@ def _import_delegations(self, parent_role): logger.debug('Adding roles delegated from ' + repr(parent_role) + '.') # Iterate the keys of the delegated roles of 'parent_role' and load them. - for keyid, keyinfo in six.iteritems(keys_info): - if keyinfo['keytype'] in ['rsa', 'ed25519', 'ecdsa-sha2-nistp256']: - - # We specify the keyid to ensure that it's the correct keyid - # for the key. - try: - - # The repo may have used hashing algorithms for the generated keyids - # that doesn't match the client's set of hash algorithms. Make sure - # to only used the repo's selected hashing algorithms. - hash_algorithms = securesystemslib.settings.HASH_ALGORITHMS - securesystemslib.settings.HASH_ALGORITHMS = keyinfo['keyid_hash_algorithms'] - key, keyids = securesystemslib.keys.format_metadata_to_key(keyinfo) - securesystemslib.settings.HASH_ALGORITHMS = hash_algorithms - - for key_id in keyids: - key['keyid'] = key_id - tuf.keydb.add_key(key, keyid=None, repository_name=self.repository_name) - - except tuf.exceptions.KeyAlreadyExistsError: - pass - - except (securesystemslib.exceptions.FormatError, securesystemslib.exceptions.Error): - logger.exception('Invalid key for keyid: ' + repr(keyid) + '.') - logger.error('Aborting role delegation for parent role ' + parent_role + '.') - raise - - else: - logger.warning('Invalid key type for ' + repr(keyid) + '.') - continue + try: + tuf.keydb.create_keydb_from_targets_metadata(parent_role, self.repository_name) + except tuf.exceptions.KeyAlreadyExistsError: + pass + except (securesystemslib.exceptions.FormatError, securesystemslib.exceptions.Error): + logger.exception('Invalid key for keyid: ' + repr(keyid) + '.') + logger.error('Aborting role delegation for parent role ' + parent_role + '.') + raise # Add the roles to the role database. for roleinfo in roles_info: @@ -984,7 +962,7 @@ def _import_delegations(self, parent_role): # is None. rolename = roleinfo.get('name') logger.debug('Adding delegated role: ' + str(rolename) + '.') - tuf.roledb.add_role(rolename, roleinfo, self.repository_name) + tuf.roledb.add_role(rolename, roleinfo, self.repository_name, parent_role) except tuf.exceptions.RoleAlreadyExistsError: logger.warning('Role already exists: ' + rolename) @@ -1380,7 +1358,7 @@ def verify_target_file(target_file_object): def _verify_uncompressed_metadata_file(self, metadata_file_object, - metadata_role): + metadata_role, delegating_rolename='root'): """ Non-public method that verifies an uncompressed metadata file. An @@ -1443,7 +1421,7 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, # Verify the signature on the downloaded metadata object. valid = tuf.sig.verify(metadata_signable, metadata_role, - self.repository_name) + self.repository_name, delegating_rolename) if not valid: raise securesystemslib.exceptions.BadSignatureError(metadata_role) @@ -1453,7 +1431,7 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, def _get_metadata_file(self, metadata_role, remote_filename, - upperbound_filelength, expected_version): + upperbound_filelength, expected_version, delegating_rolename='root'): """ Non-public method that tries downloading, up to a certain length, a @@ -1578,7 +1556,8 @@ def _get_metadata_file(self, metadata_role, remote_filename, except KeyError: logger.info(metadata_role + ' not available locally.') - self._verify_uncompressed_metadata_file(file_object, metadata_role) + self._verify_uncompressed_metadata_file(file_object, metadata_role, + delegating_metadata) except Exception as exception: # Remember the error from this mirror, and "reset" the target file. @@ -1709,7 +1688,8 @@ def _get_file(self, filepath, verify_file_function, file_type, file_length, - def _update_metadata(self, metadata_role, upperbound_filelength, version=None): + def _update_metadata(self, metadata_role, upperbound_filelength, version=None, + delegating_metadata='root'): """ Non-public method that downloads, verifies, and 'installs' the metadata @@ -1779,7 +1759,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None): metadata_file_object = \ self._get_metadata_file(metadata_role, remote_filename, - upperbound_filelength, version) + upperbound_filelength, version, delegating_metadata) # The metadata has been verified. Move the metadata file into place. # First, move the 'current' metadata file to the 'previous' directory @@ -1828,7 +1808,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None): def _update_metadata_if_changed(self, metadata_role, - referenced_metadata='snapshot'): + referenced_metadata='snapshot', delegating_rolename='root'): """ Non-public method that updates the metadata for 'metadata_role' if it has @@ -1944,7 +1924,7 @@ def _update_metadata_if_changed(self, metadata_role, try: self._update_metadata(metadata_role, upperbound_filelength, - expected_versioninfo['version']) + expected_versioninfo['version'], delegating_rolename) except Exception: # The current metadata we have is not current but we couldn't get new @@ -2517,7 +2497,9 @@ def _refresh_targets_metadata(self, rolename='targets', self._load_metadata_from_file('previous', rolename) self._load_metadata_from_file('current', rolename) - self._update_metadata_if_changed(rolename) + roleinfo = tuf.roledb.get_roleinfo(rolename, self.repository_name) + delegating_rolename = roleinfo['delegating_rolename'] + self._update_metadata_if_changed(rolename, delegating_rolename) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 0db06fe847..c831aba732 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -256,6 +256,9 @@ def write(self, write_partial=False): delegated_filename = os.path.join(self.metadata_directory, delegated_rolename + METADATA_EXTENSION) + roleinfo = tuf.roledb.get_roleinfo(delegated_rolename, repository_name) + delegating_rolename = roleinfo['delegating_rolename'] + # Ensure the parent directories of 'metadata_filepath' exist, otherwise an # IO exception is raised if 'metadata_filepath' is written to a # sub-directory. @@ -263,7 +266,7 @@ def write(self, write_partial=False): _generate_and_write_metadata(delegated_rolename, delegated_filename, write_partial, self.targets_directory, prefix=self.prefix, - repository_name=self.repository_name) + repository_name=self.repository_name, delegating_rolename) # Generate the 'project_name' metadata file. @@ -375,11 +378,15 @@ def status(self): insufficient_keys.append(delegated_role) continue + roleinfo = tuf.roledb.get_roleinfo(delegated_role, self.repository_name) + delegating_rolename = roleinfo['delegating_rolename'] + try: signable = _generate_and_write_metadata(delegated_role, filenames['targets'], False, targets_directory, False, - repository_name=self.repository_name) - self._log_status(delegated_role, signable[0], self.repository_name) + repository_name=self.repository_name, delegating_rolename) + self._log_status(delegated_role, signable[0], self.repository_name, + delegating_rolename) except securesystemslib.exceptions.Error: insufficient_signatures.append(delegated_role) @@ -423,13 +430,13 @@ def status(self): - def _log_status(self, rolename, signable, repository_name): + def _log_status(self, rolename, signable, repository_name, delegating_rolename='root'): """ Non-public function prints the number of (good/threshold) signatures of 'rolename'. """ - status = tuf.sig.get_signature_status(signable, rolename, repository_name) + status = tuf.sig.get_signature_status(signable, rolename, repository_name, delegating_rolename) message = repr(rolename) + ' role contains ' +\ repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) +\ @@ -441,7 +448,7 @@ def _log_status(self, rolename, signable, repository_name): def _generate_and_write_metadata(rolename, metadata_filename, write_partial, - targets_directory, prefix='', repository_name='default'): + targets_directory, prefix='', repository_name='default', delegating_rolename = 'root'): """ Non-public function that can generate and write the metadata of the specified 'rolename'. It also increments version numbers if: @@ -472,7 +479,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, del(metadata['targets'][element]) signable = repo_lib.sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename, repository_name) + metadata_filename, repository_name, delegating_rolename) # Check if the version number of 'rolename' may be automatically incremented, # depending on whether if partial metadata is loaded or if the metadata is @@ -480,26 +487,27 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, # Increment the version number if this is the first partial write. if write_partial: temp_signable = repo_lib.sign_metadata(metadata, [], metadata_filename, - repository_name) + repository_name, delegating_rolename) temp_signable['signatures'].extend(roleinfo['signatures']) status = tuf.sig.get_signature_status(temp_signable, rolename, - repository_name) + repository_name, delegating_rolename) if len(status['good_sigs']) == 0: metadata['version'] = metadata['version'] + 1 signable = repo_lib.sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename, repository_name) + metadata_filename, repository_name, delegating_rolename) # non-partial write() else: - if tuf.sig.verify(signable, rolename, repository_name): + if tuf.sig.verify(signable, rolename, repository_name, delegating_rolename): metadata['version'] = metadata['version'] + 1 signable = repo_lib.sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename, repository_name) + metadata_filename, repository_name, delegating_rolename) # Write the metadata to file if contains a threshold of signatures. signable['signatures'].extend(roleinfo['signatures']) - if tuf.sig.verify(signable, rolename, repository_name) or write_partial: + if tuf.sig.verify(signable, rolename, repository_name, delegating_rolename) + or write_partial: repo_lib._remove_invalid_and_duplicate_signatures(signable, repository_name) storage_backend = securesystemslib.storage.FilesystemBackend() filename = repo_lib.write_metadata_file(signable, metadata_filename, @@ -881,9 +889,8 @@ def load_project(project_directory, prefix='', new_targets_location=None, tuf.roledb.update_roleinfo(project_name, roleinfo, mark_role_as_dirty=False, repository_name=repository_name) - for key_metadata in targets_metadata['delegations']['keys'].values(): - key_object, junk = securesystemslib.keys.format_metadata_to_key(key_metadata) - tuf.keydb.add_key(key_object, repository_name=repository_name) + tuf.keydb.create_keydb_from_targets_metadata(targets_metadata['delegations'], + repository_name + ' Targets') for role in targets_metadata['delegations']['roles']: rolename = role['name'] @@ -959,14 +966,12 @@ def load_project(project_directory, prefix='', new_targets_location=None, targets_object._delegated_roles[metadata_name] = new_targets_object # Add the keys specified in the delegations field of the Targets role. - for key_metadata in metadata_object['delegations']['keys'].values(): - key_object, junk = securesystemslib.keys.format_metadata_to_key(key_metadata) - - try: - tuf.keydb.add_key(key_object, repository_name=repository_name) + try: + tuf.keydb.create_keydb_from_targets_metadata(metadata_object['delegations'], + repository_name + ' ' + metadata_name) - except tuf.exceptions.KeyAlreadyExistsError: - pass + except tuf.exceptions.KeyAlreadyExistsError: + pass for role in metadata_object['delegations']['roles']: rolename = role['name'] diff --git a/tuf/keydb.py b/tuf/keydb.py index 8011719f6d..f69c5372ca 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -152,6 +152,96 @@ def create_keydb_from_root_metadata(root_metadata, repository_name='default'): +def create_keydb_from_targets_metadata(delegations_metadata, repository_name): + """ + + Populate the key database with the unique keys found in 'delegations_metadata'. + The database dictionary will conform to + 'tuf.formats.KEYDB_SCHEMA' and have the form: {keyid: key, + ...}. The 'keyid' conforms to 'securesystemslib.formats.KEYID_SCHEMA' and + 'key' to its respective type. In the case of RSA keys, this object would + match 'RSAKEY_SCHEMA'. + + + targets_metadata: + A dictionary conformant to 'tuf.formats.DELEGATIONS_SCHEMA'. The keys found + in the 'keys' field of 'delegations_metadata' are needed by this function. + + repository_name: + The name of the repository to store the key information. + + + securesystemslib.exceptions.FormatError, if 'delegations_metadata' does not have the correct format. + + securesystemslib.exceptions.InvalidNameError, if 'repository_name' does not exist in the key + database. + + + A function to add the key to the database is called. In the case of RSA + keys, this function is add_key(). + + The old keydb key database is replaced. + + + None. + """ + + # Does 'delegations_metadata' have the correct format? + # This check will ensure 'delegations_metadata' has the appropriate number of objects + # and object types, and that all dict keys are properly named. + # Raise 'securesystemslib.exceptions.FormatError' if the check fails. + tuf.formats.DELEGATIONS_SCHEMA.check_match(delegations_metadata) + + # Does 'repository_name' have the correct format? + securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) + + # Clear the key database for 'repository_name', or create it if non-existent. + if repository_name in _keydb_dict: + _keydb_dict[repository_name].clear() + + else: + create_keydb(repository_name) + + # Iterate the keys found in 'delegations_metadata' by converting them to + # 'RSAKEY_SCHEMA' if their type is 'rsa', and then adding them to the + # key database. + for junk, key_metadata in six.iteritems(delegations_metadata['keys']): + if key_metadata['keytype'] in _SUPPORTED_KEY_TYPES: + # 'key_metadata' is stored in 'KEY_SCHEMA' format. Call + # create_from_metadata_format() to get the key in 'RSAKEY_SCHEMA' format, + # which is the format expected by 'add_key()'. Note: The 'keyids' + # returned by format_metadata_to_key() include keyids in addition to the + # default keyid listed in 'key_dict'. The additional keyids are + # generated according to securesystemslib.settings.HASH_ALGORITHMS. + + # The repo may have used hashing algorithms for the generated keyids that + # doesn't match the client's set of hash algorithms. Make sure to only + # used the repo's selected hashing algorithms. + hash_algorithms = securesystemslib.settings.HASH_ALGORITHMS + securesystemslib.settings.HASH_ALGORITHMS = key_metadata['keyid_hash_algorithms'] + key_dict, keyids = securesystemslib.keys.format_metadata_to_key(key_metadata) + securesystemslib.settings.HASH_ALGORITHMS = hash_algorithms + + try: + for keyid in keyids: + # Make sure to update key_dict['keyid'] to use one of the other valid + # keyids, otherwise add_key() will have no reference to it. + key_dict['keyid'] = keyid + add_key(key_dict, keyid=None, repository_name=repository_name) + + # Although keyid duplicates should *not* occur (unique dict keys), log a + # warning and continue. However, 'key_dict' may have already been + # adding to the keydb elsewhere. + except tuf.exceptions.KeyAlreadyExistsError as e: # pragma: no cover + logger.warning(e) + continue + + else: + logger.warning('Root Metadata file contains a key with an invalid keytype.') + + + + def create_keydb(repository_name): """ @@ -303,7 +393,7 @@ def add_key(key_dict, keyid=None, repository_name='default'): -def get_key(keyid, repository_name='default'): +def get_key(keyid, repository_name='default', delegating_rolename='root'): """ Return the key belonging to 'keyid'. @@ -317,6 +407,10 @@ def get_key(keyid, repository_name='default'): The name of the repository to get the key. If not supplied, the key is retrieved from the 'default' repository. + delegating_rolename: + The name of the delegating role to determine which keydb to access. If + not supplied, the key is retrieved from root. + securesystemslib.exceptions.FormatError, if the arguments do not have the correct format. @@ -347,6 +441,8 @@ def get_key(keyid, repository_name='default'): ' ' + repr(repository_name)) # Return the key belonging to 'keyid', if found in the key database. + if delegating_rolename is not 'root': + repository_name = repository_name + ' ' + delegating_rolename try: return copy.deepcopy(_keydb_dict[repository_name][keyid]) diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index fe77ad8413..55f969c8d3 100644 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -194,7 +194,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, # signature(s), since it can only be considered fully signed depending on # the delegating role. signable = sign_metadata(metadata, signing_keyids, metadata_filename, - repository_name) + repository_name, delegating_rolename) def should_write(): @@ -213,7 +213,7 @@ def should_write(): if should_write(): - _remove_invalid_and_duplicate_signatures(signable, repository_name) + _remove_invalid_and_duplicate_signatures(signable, repository_name, rolename) # Root should always be written as if consistent_snapshot is True (i.e., # write .root.json and root.json to disk). @@ -239,8 +239,8 @@ def should_write(): # signed, and thus its signatures should not be verified. else: signable = sign_metadata(metadata, signing_keyids, metadata_filename, - repository_name) - _remove_invalid_and_duplicate_signatures(signable, repository_name) + repository_name, delegating_rolename) + _remove_invalid_and_duplicate_signatures(signable, repository_name, rolename) # Root should always be written as if consistent_snapshot is True (i.e., # .root.json and root.json). @@ -259,7 +259,7 @@ def should_write(): -def _metadata_is_partially_loaded(rolename, signable, repository_name): +def _metadata_is_partially_loaded(rolename, signable, repository_name, delegating_rolename='root'): """ Non-public function that determines whether 'rolename' is loaded with at least zero good signatures, but an insufficient threshold (which means @@ -276,7 +276,8 @@ def _metadata_is_partially_loaded(rolename, signable, repository_name): # The signature status lists the number of good signatures, including # bad, untrusted, unknown, etc. - status = tuf.sig.get_signature_status(signable, rolename, repository_name) + status = tuf.sig.get_signature_status(signable, rolename, repository_name, + delegating_rolename) if len(status['good_sigs']) < status['threshold'] and \ len(status['good_sigs']) >= 0: @@ -317,7 +318,7 @@ def _check_role_keys(rolename, repository_name): -def _remove_invalid_and_duplicate_signatures(signable, repository_name): +def _remove_invalid_and_duplicate_signatures(signable, repository_name, rolename): """ Non-public function that removes invalid or duplicate signatures from 'signable'. 'signable' may contain signatures (invalid) from previous @@ -338,6 +339,8 @@ def _remove_invalid_and_duplicate_signatures(signable, repository_name): # Remove 'signature' from 'signable' if the listed keyid does not exist # in 'tuf.keydb'. + if rolename not in tuf.roledb.TOP_LEVEL_ROLES: + repository_name = repository_name + ' ' + rolename try: key = tuf.keydb.get_key(keyid, repository_name=repository_name) @@ -641,31 +644,10 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): tuf.roledb.update_roleinfo('targets', roleinfo, mark_role_as_dirty=False, repository_name=repository_name) - # Add the keys specified in the delegations field of the Targets role. - for key_metadata in six.itervalues(targets_metadata['delegations']['keys']): - - # The repo may have used hashing algorithms for the generated keyids - # that doesn't match the client's set of hash algorithms. Make sure - # to only used the repo's selected hashing algorithms. - hash_algorithms = securesystemslib.settings.HASH_ALGORITHMS - securesystemslib.settings.HASH_ALGORITHMS = key_metadata['keyid_hash_algorithms'] - key_object, keyids = securesystemslib.keys.format_metadata_to_key(key_metadata) - securesystemslib.settings.HASH_ALGORITHMS = hash_algorithms - - # Add 'key_object' to the list of recognized keys. Keys may be shared, - # so do not raise an exception if 'key_object' has already been loaded. - # In contrast to the methods that may add duplicate keys, do not log - # a warning as there may be many such duplicate key warnings. The - # repository maintainer should have also been made aware of the duplicate - # key when it was added. - try: - for keyid in keyids: #pragma: no branch - key_object['keyid'] = keyid - tuf.keydb.add_key(key_object, keyid=None, - repository_name=repository_name) - - except tuf.exceptions.KeyAlreadyExistsError: - pass + # Create a keydb for the keys specified in the delegations field of the Targets + # role. This keydb will be used for all delegations from the Target role. + create_keydb_from_targets_metadata(targets_metadata['delegations'], + repository_name + ' Targets') except securesystemslib.exceptions.StorageError: raise tuf.exceptions.RepositoryError('The Targets file can not be loaded: ' @@ -1784,7 +1766,7 @@ def generate_timestamp_metadata(snapshot_file_path, version, expiration_date, -def sign_metadata(metadata_object, keyids, filename, repository_name): +def sign_metadata(metadata_object, keyids, filename, repository_name, rolename='root'): """ Sign a metadata object. If any of the keyids have already signed the file, @@ -1809,6 +1791,10 @@ def sign_metadata(metadata_object, keyids, filename, repository_name): The name of the repository. If not supplied, 'rolename' is added to the 'default' repository. + rolename: + The name of the signing role. This is used to find the correct keydb. + If not supplied, keys from 'root' are used. + securesystemslib.exceptions.FormatError, if a valid 'signable' object could not be generated or the arguments are improperly formatted. @@ -1838,6 +1824,9 @@ def sign_metadata(metadata_object, keyids, filename, repository_name): # keyid of 'keyids'. signable = tuf.formats.make_signable(metadata_object) + if delegating_rolename is not 'root': + repository_name = repository_name + ' ' + delegating_rolename + # Sign the metadata with each keyid in 'keyids'. 'signable' should have # zero signatures (metadata_object contained none). for keyid in keyids: @@ -2124,13 +2113,13 @@ def _log_status_of_top_level_roles(targets_directory, metadata_directory, -def _log_status(rolename, signable, repository_name): +def _log_status(rolename, signable, repository_name, delegating_rolename='root'): """ Non-public function logs the number of (good/threshold) signatures of 'rolename'. """ - status = tuf.sig.get_signature_status(signable, rolename, repository_name) + status = tuf.sig.get_signature_status(signable, rolename, repository_name, delegating_rolename) logger.info(repr(rolename) + ' role contains ' + \ repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) + \ diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 3a2f4dce28..48d343526d 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -3182,25 +3182,11 @@ def load_repository(repository_directory, repository_name='default', # log a warning here as there may be many such duplicate key warnings. # The repository maintainer should have also been made aware of the # duplicate key when it was added. - for key_metadata in six.itervalues(metadata_object['delegations']['keys']): - - # The repo may have used hashing algorithms for the generated keyids - # that doesn't match the client's set of hash algorithms. Make sure - # to only used the repo's selected hashing algorithms. - hash_algorithms = securesystemslib.settings.HASH_ALGORITHMS - securesystemslib.settings.HASH_ALGORITHMS = \ - key_metadata['keyid_hash_algorithms'] - key_object, keyids = \ - securesystemslib.keys.format_metadata_to_key(key_metadata) - securesystemslib.settings.HASH_ALGORITHMS = hash_algorithms - try: - for keyid in keyids: # pragma: no branch - key_object['keyid'] = keyid - tuf.keydb.add_key(key_object, keyid=None, - repository_name=repository_name) - - except tuf.exceptions.KeyAlreadyExistsError: - pass + try: + tuf.keydb.create_keydb_from_targets_metadata(metadata_object['delegations'], + repository_name + ' ' + rolename) + except tuf.exceptions.KeyAlreadyExistsError: + pass return repository diff --git a/tuf/sig.py b/tuf/sig.py index 2494765bf2..cd02bcfcbe 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -67,7 +67,7 @@ def get_signature_status(signable, role=None, repository_name='default', - threshold=None, keyids=None): + threshold=None, keyids=None, delegating_rolename='root'): """ Return a dictionary representing the status of the signatures listed in @@ -163,7 +163,7 @@ def get_signature_status(signable, role=None, repository_name='default', # Does the signature use an unrecognized key? try: - key = tuf.keydb.get_key(keyid, repository_name) + key = tuf.keydb.get_key(keyid, repository_name, delegating_rolename) except tuf.exceptions.UnknownKeyError: unknown_sigs.append(keyid) @@ -233,7 +233,7 @@ def get_signature_status(signable, role=None, repository_name='default', def verify(signable, role, repository_name='default', threshold=None, - keyids=None): + keyids=None, delegating_rolename='root'): """ Verify that 'signable' has a valid threshold of authorized signatures @@ -293,7 +293,8 @@ def verify(signable, role, repository_name='default', threshold=None, # tuf.exceptions.UnknownRoleError # securesystemslib.exceptions.FormatError. 'threshold' and 'keyids' are also # validated. - status = get_signature_status(signable, role, repository_name, threshold, keyids) + status = get_signature_status(signable, role, repository_name, threshold, keyids, + delegating_rolename) # Retrieve the role's threshold and the authorized keys of 'status' threshold = status['threshold'] From 043c05639428875aa62501ad3db4f2087b3d48f9 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 3 Aug 2020 11:11:30 -0700 Subject: [PATCH 02/10] This commit adds the parent_role field to roledb. This field can be used by delegated targets to determine which role delegated to them, and thus which keys should be used to verify the role. Signed-off-by: marinamoore --- tuf/client/updater.py | 3 ++- tuf/developer_tool.py | 6 ++++-- tuf/formats.py | 3 ++- tuf/roledb.py | 19 +++++++++++-------- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index bdd0dc3b61..0695c2d82a 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -962,7 +962,8 @@ def _import_delegations(self, parent_role): # is None. rolename = roleinfo.get('name') logger.debug('Adding delegated role: ' + str(rolename) + '.') - tuf.roledb.add_role(rolename, roleinfo, self.repository_name, parent_role) + roleinfo['parent_role'] = parent_role + tuf.roledb.add_role(rolename, roleinfo, self.repository_name) except tuf.exceptions.RoleAlreadyExistsError: logger.warning('Role already exists: ' + rolename) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index c831aba732..a357cb1da9 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -897,7 +897,8 @@ def load_project(project_directory, prefix='', new_targets_location=None, roleinfo = {'name': role['name'], 'keyids': role['keyids'], 'threshold': role['threshold'], 'signing_keyids': [], 'signatures': [], 'partial_loaded':False, - 'delegations': {'keys':{}, 'roles':[]} + 'delegations': {'keys':{}, 'roles':[]}, + 'parent_role': targets_metadata['name'] } tuf.roledb.add_role(rolename, roleinfo, repository_name=repository_name) @@ -980,7 +981,8 @@ def load_project(project_directory, prefix='', new_targets_location=None, 'signing_keyids': [], 'signatures': [], 'partial_loaded': False, 'delegations': {'keys': {}, - 'roles': []}} + 'roles': []}, + 'parent_role' : metadata_object['name']} tuf.roledb.add_role(rolename, roleinfo, repository_name=repository_name) if new_prefix: diff --git a/tuf/formats.py b/tuf/formats.py index caa1490256..b4d35c31ee 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -137,7 +137,8 @@ threshold = THRESHOLD_SCHEMA, terminating = SCHEMA.Optional(securesystemslib.formats.BOOLEAN_SCHEMA), paths = SCHEMA.Optional(RELPATHS_SCHEMA), - path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA)) + path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA), + parent_role = SCHEMA.Optional(ROLENAME_SCHEMA)) # A dict of roles where the dict keys are role names and the dict values holding # the role data/information. diff --git a/tuf/roledb.py b/tuf/roledb.py index 37add72e3a..7e37a0423f 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -35,10 +35,11 @@ 'signatures': ['abcd3452...'], 'paths': ['role.json'], 'path_hash_prefixes': ['ab34df13'], - 'delegations': {'keys': {}, 'roles': {}}} + 'delegations': {'keys': {}, 'roles': {}, + 'parent_role': 'parent_rolename'}} - The 'name', 'paths', 'path_hash_prefixes', and 'delegations' dict keys are - optional. + The 'name', 'paths', 'path_hash_prefixes', 'delegations', and 'parent_role' + dict keys are optional. """ # Help with Python 3 compatibility, where the print statement is a function, an @@ -263,9 +264,10 @@ def add_role(rolename, roleinfo, repository_name='default'): 'paths': ['path/to/target1', 'path/to/target2', ...], 'path_hash_prefixes': ['a324fcd...', ...], 'delegations': {'keys': } + 'parent_role' " 'parent_rolename'} - The 'paths', 'path_hash_prefixes', and 'delegations' dict keys are - optional. + The 'paths', 'path_hash_prefixes', 'delegations', and 'parent_role' + dict keys are optional. The 'target' role has an additional 'paths' key. Its value is a list of strings representing the path of the target file(s). @@ -693,10 +695,11 @@ def get_roleinfo(rolename, repository_name='default'): 'signatures': ['ab453bdf...', ...], 'paths': ['path/to/target1', 'path/to/target2', ...], 'path_hash_prefixes': ['a324fcd...', ...], - 'delegations': {'keys': {}, 'roles': []}} + 'delegations': {'keys': {}, 'roles': []} + 'parent_role' : 'parent_rolename'} - The 'signatures', 'paths', 'path_hash_prefixes', and 'delegations' dict keys - are optional. + The 'signatures', 'paths', 'path_hash_prefixes', 'delegations', and + 'parent_role' dict keys are optional. rolename: From de40f852b959ce3e86d9b7c0021eff1a5a4f06dd Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 3 Aug 2020 13:30:12 -0700 Subject: [PATCH 03/10] Bug fixes for the per client keydb Signed-off-by: marinamoore --- tuf/client/updater.py | 22 +++++++++++-------- tuf/developer_tool.py | 48 ++++++++++++++++++++++++------------------ tuf/keydb.py | 5 ++++- tuf/repository_lib.py | 37 +++++++++++++++++++++----------- tuf/repository_tool.py | 2 +- 5 files changed, 71 insertions(+), 43 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 0695c2d82a..47ab8093ee 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -947,11 +947,11 @@ def _import_delegations(self, parent_role): # Iterate the keys of the delegated roles of 'parent_role' and load them. try: - tuf.keydb.create_keydb_from_targets_metadata(parent_role, self.repository_name) + tuf.keydb.create_keydb_from_targets_metadata(current_parent_metadata['delegations'], + self.repository_name, parent_role) except tuf.exceptions.KeyAlreadyExistsError: pass except (securesystemslib.exceptions.FormatError, securesystemslib.exceptions.Error): - logger.exception('Invalid key for keyid: ' + repr(keyid) + '.') logger.error('Aborting role delegation for parent role ' + parent_role + '.') raise @@ -1422,7 +1422,7 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, # Verify the signature on the downloaded metadata object. valid = tuf.sig.verify(metadata_signable, metadata_role, - self.repository_name, delegating_rolename) + self.repository_name, delegating_rolename=delegating_rolename) if not valid: raise securesystemslib.exceptions.BadSignatureError(metadata_role) @@ -1558,7 +1558,7 @@ def _get_metadata_file(self, metadata_role, remote_filename, logger.info(metadata_role + ' not available locally.') self._verify_uncompressed_metadata_file(file_object, metadata_role, - delegating_metadata) + delegating_rolename=delegating_rolename) except Exception as exception: # Remember the error from this mirror, and "reset" the target file. @@ -1690,7 +1690,7 @@ def _get_file(self, filepath, verify_file_function, file_type, file_length, def _update_metadata(self, metadata_role, upperbound_filelength, version=None, - delegating_metadata='root'): + delegating_rolename='root'): """ Non-public method that downloads, verifies, and 'installs' the metadata @@ -1760,7 +1760,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, metadata_file_object = \ self._get_metadata_file(metadata_role, remote_filename, - upperbound_filelength, version, delegating_metadata) + upperbound_filelength, version, delegating_rolename=delegating_rolename) # The metadata has been verified. Move the metadata file into place. # First, move the 'current' metadata file to the 'previous' directory @@ -1925,7 +1925,7 @@ def _update_metadata_if_changed(self, metadata_role, try: self._update_metadata(metadata_role, upperbound_filelength, - expected_versioninfo['version'], delegating_rolename) + expected_versioninfo['version'], delegating_rolename=delegating_rolename) except Exception: # The current metadata we have is not current but we couldn't get new @@ -2499,8 +2499,12 @@ def _refresh_targets_metadata(self, rolename='targets', self._load_metadata_from_file('current', rolename) roleinfo = tuf.roledb.get_roleinfo(rolename, self.repository_name) - delegating_rolename = roleinfo['delegating_rolename'] - self._update_metadata_if_changed(rolename, delegating_rolename) + try: + delegating_rolename = roleinfo['parent_role'] + except: + delegating_rolename = 'root' + self._update_metadata_if_changed(rolename, + delegating_rolename=delegating_rolename) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index a357cb1da9..af7ab2deb6 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -256,8 +256,11 @@ def write(self, write_partial=False): delegated_filename = os.path.join(self.metadata_directory, delegated_rolename + METADATA_EXTENSION) - roleinfo = tuf.roledb.get_roleinfo(delegated_rolename, repository_name) - delegating_rolename = roleinfo['delegating_rolename'] + roleinfo = tuf.roledb.get_roleinfo(delegated_rolename, self.repository_name) + try: + delegating_rolename = roleinfo['parent_role'] + except: + delegating_rolename = 'root' # Ensure the parent directories of 'metadata_filepath' exist, otherwise an # IO exception is raised if 'metadata_filepath' is written to a @@ -266,7 +269,7 @@ def write(self, write_partial=False): _generate_and_write_metadata(delegated_rolename, delegated_filename, write_partial, self.targets_directory, prefix=self.prefix, - repository_name=self.repository_name, delegating_rolename) + repository_name=self.repository_name, delegating_rolename=delegating_rolename) # Generate the 'project_name' metadata file. @@ -379,14 +382,18 @@ def status(self): continue roleinfo = tuf.roledb.get_roleinfo(delegated_role, self.repository_name) - delegating_rolename = roleinfo['delegating_rolename'] + try: + delegating_rolename = roleinfo['parent_role'] + except: + delegating_rolename = 'root' try: signable = _generate_and_write_metadata(delegated_role, filenames['targets'], False, targets_directory, False, - repository_name=self.repository_name, delegating_rolename) + repository_name=self.repository_name, + delegating_rolename=delegating_rolename) self._log_status(delegated_role, signable[0], self.repository_name, - delegating_rolename) + delegating_rolename=delegating_rolename) except securesystemslib.exceptions.Error: insufficient_signatures.append(delegated_role) @@ -436,7 +443,7 @@ def _log_status(self, rolename, signable, repository_name, delegating_rolename=' 'rolename'. """ - status = tuf.sig.get_signature_status(signable, rolename, repository_name, delegating_rolename) + status = tuf.sig.get_signature_status(signable, rolename, repository_name, delegating_rolename=delegating_rolename) message = repr(rolename) + ' role contains ' +\ repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) +\ @@ -479,7 +486,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, del(metadata['targets'][element]) signable = repo_lib.sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename, repository_name, delegating_rolename) + metadata_filename, repository_name, delegating_rolename=delegating_rolename) # Check if the version number of 'rolename' may be automatically incremented, # depending on whether if partial metadata is loaded or if the metadata is @@ -487,28 +494,29 @@ def _generate_and_write_metadata(rolename, metadata_filename, write_partial, # Increment the version number if this is the first partial write. if write_partial: temp_signable = repo_lib.sign_metadata(metadata, [], metadata_filename, - repository_name, delegating_rolename) + repository_name, delegating_rolename=delegating_rolename) temp_signable['signatures'].extend(roleinfo['signatures']) status = tuf.sig.get_signature_status(temp_signable, rolename, - repository_name, delegating_rolename) + repository_name, delegating_rolename=delegating_rolename) if len(status['good_sigs']) == 0: metadata['version'] = metadata['version'] + 1 signable = repo_lib.sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename, repository_name, delegating_rolename) + metadata_filename, repository_name, delegating_rolename=delegating_rolename) # non-partial write() else: - if tuf.sig.verify(signable, rolename, repository_name, delegating_rolename): + if tuf.sig.verify(signable, rolename, repository_name, + delegating_rolename=delegating_rolename): metadata['version'] = metadata['version'] + 1 signable = repo_lib.sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename, repository_name, delegating_rolename) + metadata_filename, repository_name, delegating_rolename=delegating_rolename) # Write the metadata to file if contains a threshold of signatures. signable['signatures'].extend(roleinfo['signatures']) - if tuf.sig.verify(signable, rolename, repository_name, delegating_rolename) - or write_partial: - repo_lib._remove_invalid_and_duplicate_signatures(signable, repository_name) + if tuf.sig.verify(signable, rolename, repository_name, + delegating_rolename=delegating_rolename) or write_partial: + repo_lib._remove_invalid_and_duplicate_signatures(signable, repository_name, delegating_rolename=delegating_rolename) storage_backend = securesystemslib.storage.FilesystemBackend() filename = repo_lib.write_metadata_file(signable, metadata_filename, metadata['version'], False, storage_backend) @@ -890,7 +898,7 @@ def load_project(project_directory, prefix='', new_targets_location=None, repository_name=repository_name) tuf.keydb.create_keydb_from_targets_metadata(targets_metadata['delegations'], - repository_name + ' Targets') + repository_name, 'Targets') for role in targets_metadata['delegations']['roles']: rolename = role['name'] @@ -898,7 +906,7 @@ def load_project(project_directory, prefix='', new_targets_location=None, 'threshold': role['threshold'], 'signing_keyids': [], 'signatures': [], 'partial_loaded':False, 'delegations': {'keys':{}, 'roles':[]}, - 'parent_role': targets_metadata['name'] + 'parent_role': 'Targets' } tuf.roledb.add_role(rolename, roleinfo, repository_name=repository_name) @@ -969,7 +977,7 @@ def load_project(project_directory, prefix='', new_targets_location=None, # Add the keys specified in the delegations field of the Targets role. try: tuf.keydb.create_keydb_from_targets_metadata(metadata_object['delegations'], - repository_name + ' ' + metadata_name) + repository_name, metadata_name) except tuf.exceptions.KeyAlreadyExistsError: pass @@ -982,7 +990,7 @@ def load_project(project_directory, prefix='', new_targets_location=None, 'partial_loaded': False, 'delegations': {'keys': {}, 'roles': []}, - 'parent_role' : metadata_object['name']} + 'parent_role' : metadata_name} tuf.roledb.add_role(rolename, roleinfo, repository_name=repository_name) if new_prefix: diff --git a/tuf/keydb.py b/tuf/keydb.py index f69c5372ca..b5f855151f 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -152,7 +152,8 @@ def create_keydb_from_root_metadata(root_metadata, repository_name='default'): -def create_keydb_from_targets_metadata(delegations_metadata, repository_name): +def create_keydb_from_targets_metadata(delegations_metadata, repository_name, + delegating_rolename): """ Populate the key database with the unique keys found in 'delegations_metadata'. @@ -195,6 +196,8 @@ def create_keydb_from_targets_metadata(delegations_metadata, repository_name): # Does 'repository_name' have the correct format? securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) + repository_name = repository_name + ' ' + delegating_rolename + # Clear the key database for 'repository_name', or create it if non-existent. if repository_name in _keydb_dict: _keydb_dict[repository_name].clear() diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index 55f969c8d3..1877f03953 100644 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -193,8 +193,13 @@ def _generate_and_write_metadata(rolename, metadata_filename, # role should not be written to disk without full verification of its # signature(s), since it can only be considered fully signed depending on # the delegating role. + roleinfo = tuf.roledb.get_roleinfo(rolename, repository_name) + try: + delegating_rolename = roleinfo['parent_role'] + except: + delegating_rolename = 'root' signable = sign_metadata(metadata, signing_keyids, metadata_filename, - repository_name, delegating_rolename) + repository_name, delegating_rolename=delegating_rolename) def should_write(): @@ -213,7 +218,8 @@ def should_write(): if should_write(): - _remove_invalid_and_duplicate_signatures(signable, repository_name, rolename) + _remove_invalid_and_duplicate_signatures(signable, repository_name, + delegating_rolename=delegating_rolename) # Root should always be written as if consistent_snapshot is True (i.e., # write .root.json and root.json to disk). @@ -238,9 +244,14 @@ def should_write(): # 'rolename' is a delegated role or a top-level role that is partially # signed, and thus its signatures should not be verified. else: + roleinfo = tuf.roledb.get_roleinfo(rolename, repository_name) + try: + delegating_rolename = roleinfo['parent_role'] + except: + delegating_rolename = 'root' signable = sign_metadata(metadata, signing_keyids, metadata_filename, - repository_name, delegating_rolename) - _remove_invalid_and_duplicate_signatures(signable, repository_name, rolename) + repository_name, delegating_rolename=delegating_rolename) + _remove_invalid_and_duplicate_signatures(signable, repository_name, delegating_rolename=delegating_rolename) # Root should always be written as if consistent_snapshot is True (i.e., # .root.json and root.json). @@ -277,7 +288,7 @@ def _metadata_is_partially_loaded(rolename, signable, repository_name, delegatin # The signature status lists the number of good signatures, including # bad, untrusted, unknown, etc. status = tuf.sig.get_signature_status(signable, rolename, repository_name, - delegating_rolename) + delegating_rolename=delegating_rolename) if len(status['good_sigs']) < status['threshold'] and \ len(status['good_sigs']) >= 0: @@ -318,7 +329,8 @@ def _check_role_keys(rolename, repository_name): -def _remove_invalid_and_duplicate_signatures(signable, repository_name, rolename): +def _remove_invalid_and_duplicate_signatures(signable, repository_name, + delegating_rolename='root'): """ Non-public function that removes invalid or duplicate signatures from 'signable'. 'signable' may contain signatures (invalid) from previous @@ -339,8 +351,8 @@ def _remove_invalid_and_duplicate_signatures(signable, repository_name, rolename # Remove 'signature' from 'signable' if the listed keyid does not exist # in 'tuf.keydb'. - if rolename not in tuf.roledb.TOP_LEVEL_ROLES: - repository_name = repository_name + ' ' + rolename + if delegating_rolename not in tuf.roledb.TOP_LEVEL_ROLES: + repository_name = repository_name + ' ' + delegating_rolename try: key = tuf.keydb.get_key(keyid, repository_name=repository_name) @@ -646,8 +658,8 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): # Create a keydb for the keys specified in the delegations field of the Targets # role. This keydb will be used for all delegations from the Target role. - create_keydb_from_targets_metadata(targets_metadata['delegations'], - repository_name + ' Targets') + tuf.keydb.create_keydb_from_targets_metadata(targets_metadata['delegations'], + repository_name, 'Targets') except securesystemslib.exceptions.StorageError: raise tuf.exceptions.RepositoryError('The Targets file can not be loaded: ' @@ -1766,7 +1778,8 @@ def generate_timestamp_metadata(snapshot_file_path, version, expiration_date, -def sign_metadata(metadata_object, keyids, filename, repository_name, rolename='root'): +def sign_metadata(metadata_object, keyids, filename, repository_name, + delegating_rolename='root'): """ Sign a metadata object. If any of the keyids have already signed the file, @@ -2119,7 +2132,7 @@ def _log_status(rolename, signable, repository_name, delegating_rolename='root') 'rolename'. """ - status = tuf.sig.get_signature_status(signable, rolename, repository_name, delegating_rolename) + status = tuf.sig.get_signature_status(signable, rolename, repository_name, delegating_rolename=delegating_rolename) logger.info(repr(rolename) + ' role contains ' + \ repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) + \ diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 48d343526d..6dade4bd78 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -3184,7 +3184,7 @@ def load_repository(repository_directory, repository_name='default', # duplicate key when it was added. try: tuf.keydb.create_keydb_from_targets_metadata(metadata_object['delegations'], - repository_name + ' ' + rolename) + repository_name, rolename) except tuf.exceptions.KeyAlreadyExistsError: pass From 1c121e3ab99b29409d4f28f830dd372865b9a61b Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 3 Aug 2020 14:48:38 -0700 Subject: [PATCH 04/10] Update tests for local keydb. The repository keydb will no longer store keys for delegated roles. Instead look for these keys in delegated keydbs. Signed-off-by: marinamoore --- tests/test_updater.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index 9a55d3bc2f..049334ec40 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -301,7 +301,6 @@ def test_1__init__exceptions(self): def test_1__load_metadata_from_file(self): - # Setup # Get the 'role1.json' filepath. Manually load the role metadata, and # compare it against the loaded metadata by '_load_metadata_from_file()'. @@ -320,6 +319,8 @@ def test_1__load_metadata_from_file(self): self.assertEqual(len(self.repository_updater.metadata['current']), 5) # Verify that the content of root metadata is valid. + # load_json_file does not set 'parent_role' + role1_meta['signed']['delegations']['roles'][0]['parent_role'] = 'role1' self.assertEqual(self.repository_updater.metadata['current']['role1'], role1_meta['signed']) @@ -353,13 +354,14 @@ def test_1__rebuild_key_and_role_db(self): self.assertEqual(root_roleinfo['threshold'], root_threshold) - # Ensure we add 2 to the number of root keys (actually, the number of root - # keys multiplied by the number of keyid hash algorithms), to include the - # delegated targets key (+1 for its sha512 keyid). The delegated roles of - # 'targets.json' are also loaded when the repository object is - # instantiated. + # Verify that the keydb for root contains only keys for top level roles + + self.assertEqual(number_of_root_keys * 2, len(tuf.keydb._keydb_dict[self.repository_name])) - self.assertEqual(number_of_root_keys * 2 + 2, len(tuf.keydb._keydb_dict[self.repository_name])) + # Ensure that the delegated targets keys are stored in their own keydb. + # The delegated roles of 'targets.json' are also loaded when the repository + # object is instantiated. + self.assertEqual(2, len(tuf.keydb._keydb_dict[self.repository_name + ' targets'])) # Test: normal case. self.repository_updater._rebuild_key_and_role_db() @@ -583,9 +585,10 @@ def test_2__import_delegations(self): self.repository_updater._import_delegations('targets') self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 5) - # The number of root keys (times the number of key hash algorithms) + - # delegation's key (+1 for its sha512 keyid). - self.assertEqual(len(tuf.keydb._keydb_dict[repository_name]), 4 * 2 + 2) + # The number of root keys (times the number of key hash algorithms) + self.assertEqual(len(tuf.keydb._keydb_dict[repository_name]), 4 * 2) + # The delegation's key (+1 for its sha512 keyid). + self.assertEqual(len(tuf.keydb._keydb_dict[repository_name + ' targets']), 2) # Verify that roledb dictionary was added. self.assertTrue('role1' in tuf.roledb._roledb_dict[repository_name]) @@ -599,7 +602,7 @@ def test_2__import_delegations(self): keyids.append(signature['keyid']) for keyid in keyids: - self.assertTrue(keyid in tuf.keydb._keydb_dict[repository_name]) + self.assertTrue(keyid in tuf.keydb._keydb_dict[repository_name] or keyid in tuf.keydb._keydb_dict[repository_name + ' targets']) # Verify that _import_delegations() ignores invalid keytypes in the 'keys' # field of parent role's 'delegations'. @@ -2117,6 +2120,7 @@ def test_get_valid_targetinfo(self): # verify that get_valid_targetinfo() raises an UnknownTargetError # despite both repos signing for file1.txt. multi_repo_updater.map_file['mapping'][0]['threshold'] = 2 + print(multi_repo_updater.get_valid_targetinfo('file1.txt')) self.assertRaises(tuf.exceptions.UnknownTargetError, multi_repo_updater.get_valid_targetinfo, 'file1.txt') From e9a59f564f7460b8720b1f59dec64cd454c4f0f5 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 3 Aug 2020 15:40:20 -0700 Subject: [PATCH 05/10] Fix minor issues for pylint Signed-off-by: marinamoore --- tuf/client/updater.py | 3 +-- tuf/developer_tool.py | 4 ++-- tuf/keydb.py | 2 +- tuf/repository_lib.py | 4 ++-- tuf/repository_tool.py | 1 - 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 47ab8093ee..7244810eb3 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -940,7 +940,6 @@ def _import_delegations(self, parent_role): return # This could be quite slow with a large number of delegations. - keys_info = current_parent_metadata['delegations'].get('keys', {}) roles_info = current_parent_metadata['delegations'].get('roles', []) logger.debug('Adding roles delegated from ' + repr(parent_role) + '.') @@ -2501,7 +2500,7 @@ def _refresh_targets_metadata(self, rolename='targets', roleinfo = tuf.roledb.get_roleinfo(rolename, self.repository_name) try: delegating_rolename = roleinfo['parent_role'] - except: + except KeyError: delegating_rolename = 'root' self._update_metadata_if_changed(rolename, delegating_rolename=delegating_rolename) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index af7ab2deb6..dbac50c1a8 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -259,7 +259,7 @@ def write(self, write_partial=False): roleinfo = tuf.roledb.get_roleinfo(delegated_rolename, self.repository_name) try: delegating_rolename = roleinfo['parent_role'] - except: + except KeyError: delegating_rolename = 'root' # Ensure the parent directories of 'metadata_filepath' exist, otherwise an @@ -384,7 +384,7 @@ def status(self): roleinfo = tuf.roledb.get_roleinfo(delegated_role, self.repository_name) try: delegating_rolename = roleinfo['parent_role'] - except: + except KeyError: delegating_rolename = 'root' try: diff --git a/tuf/keydb.py b/tuf/keydb.py index b5f855151f..5c30061815 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -444,7 +444,7 @@ def get_key(keyid, repository_name='default', delegating_rolename='root'): ' ' + repr(repository_name)) # Return the key belonging to 'keyid', if found in the key database. - if delegating_rolename is not 'root': + if delegating_rolename != 'root': repository_name = repository_name + ' ' + delegating_rolename try: return copy.deepcopy(_keydb_dict[repository_name][keyid]) diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index 1877f03953..8a98c0f2bb 100644 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -196,7 +196,7 @@ def _generate_and_write_metadata(rolename, metadata_filename, roleinfo = tuf.roledb.get_roleinfo(rolename, repository_name) try: delegating_rolename = roleinfo['parent_role'] - except: + except KeyError: delegating_rolename = 'root' signable = sign_metadata(metadata, signing_keyids, metadata_filename, repository_name, delegating_rolename=delegating_rolename) @@ -247,7 +247,7 @@ def should_write(): roleinfo = tuf.roledb.get_roleinfo(rolename, repository_name) try: delegating_rolename = roleinfo['parent_role'] - except: + except KeyError: delegating_rolename = 'root' signable = sign_metadata(metadata, signing_keyids, metadata_filename, repository_name, delegating_rolename=delegating_rolename) diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 6dade4bd78..370c1b1cdd 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -54,7 +54,6 @@ import securesystemslib.formats import securesystemslib.util import iso8601 -import six import securesystemslib.storage From 4d866c475908e2676069c3fb7fd21debc1985217 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 3 Aug 2020 17:34:34 -0700 Subject: [PATCH 06/10] Bug fixes Signed-off-by: marinamoore --- tuf/developer_tool.py | 4 ++-- tuf/repository_lib.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index dbac50c1a8..8a12a2a6cf 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -898,7 +898,7 @@ def load_project(project_directory, prefix='', new_targets_location=None, repository_name=repository_name) tuf.keydb.create_keydb_from_targets_metadata(targets_metadata['delegations'], - repository_name, 'Targets') + repository_name, 'targets') for role in targets_metadata['delegations']['roles']: rolename = role['name'] @@ -906,7 +906,7 @@ def load_project(project_directory, prefix='', new_targets_location=None, 'threshold': role['threshold'], 'signing_keyids': [], 'signatures': [], 'partial_loaded':False, 'delegations': {'keys':{}, 'roles':[]}, - 'parent_role': 'Targets' + 'parent_role': 'targets' } tuf.roledb.add_role(rolename, roleinfo, repository_name=repository_name) diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index 8a98c0f2bb..032bb68448 100644 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -659,7 +659,7 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): # Create a keydb for the keys specified in the delegations field of the Targets # role. This keydb will be used for all delegations from the Target role. tuf.keydb.create_keydb_from_targets_metadata(targets_metadata['delegations'], - repository_name, 'Targets') + repository_name, 'targets') except securesystemslib.exceptions.StorageError: raise tuf.exceptions.RepositoryError('The Targets file can not be loaded: ' @@ -1837,7 +1837,7 @@ def sign_metadata(metadata_object, keyids, filename, repository_name, # keyid of 'keyids'. signable = tuf.formats.make_signable(metadata_object) - if delegating_rolename is not 'root': + if delegating_rolename != 'root': repository_name = repository_name + ' ' + delegating_rolename # Sign the metadata with each keyid in 'keyids'. 'signable' should have From e0c50c0230f277f4d525d388ac0a37657946fcd8 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Tue, 4 Aug 2020 10:33:55 -0700 Subject: [PATCH 07/10] use correct keydb when loading keys Signed-off-by: marinamoore --- tests/test_developer_tool.py | 2 +- tests/test_updater.py | 1 - tuf/repository_tool.py | 12 ++++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_developer_tool.py b/tests/test_developer_tool.py index b595aeda61..ac5785be70 100755 --- a/tests/test_developer_tool.py +++ b/tests/test_developer_tool.py @@ -375,7 +375,7 @@ def test_write(self): # Load_signing_keys. - project('delegation').load_signing_key(delegation_private_key) + project('delegation').load_signing_key(delegation_private_key, 'targets') project.status() diff --git a/tests/test_updater.py b/tests/test_updater.py index 049334ec40..71fd319ab0 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -2120,7 +2120,6 @@ def test_get_valid_targetinfo(self): # verify that get_valid_targetinfo() raises an UnknownTargetError # despite both repos signing for file1.txt. multi_repo_updater.map_file['mapping'][0]['threshold'] = 2 - print(multi_repo_updater.get_valid_targetinfo('file1.txt')) self.assertRaises(tuf.exceptions.UnknownTargetError, multi_repo_updater.get_valid_targetinfo, 'file1.txt') diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 370c1b1cdd..f3b9c912e6 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -853,7 +853,7 @@ def remove_verification_key(self, key): - def load_signing_key(self, key): + def load_signing_key(self, key, delegating_rolename='root'): """ Load the role key, which must contain the private portion, so that role @@ -897,11 +897,15 @@ def load_signing_key(self, key): # Has the key, with the private portion included, been added to the keydb? # The public version of the key may have been previously added. try: - tuf.keydb.add_key(key, repository_name=self._repository_name) + if delegating_rolename != 'root': + repository_name = self._repository_name + ' ' + delegating_rolename + else: + repository_name = self._repository_name + tuf.keydb.add_key(key, repository_name=repository_name) except tuf.exceptions.KeyAlreadyExistsError: - tuf.keydb.remove_key(key['keyid'], self._repository_name) - tuf.keydb.add_key(key, repository_name=self._repository_name) + tuf.keydb.remove_key(key['keyid'], repository_name) + tuf.keydb.add_key(key, repository_name=repository_name) # Update the role's 'signing_keys' field in 'tuf.roledb.py'. roleinfo = tuf.roledb.get_roleinfo(self.rolename, self._repository_name) From 5b57fa086e28af6fe0769e1ee9245872240c4e5c Mon Sep 17 00:00:00 2001 From: marinamoore Date: Tue, 4 Aug 2020 11:26:45 -0700 Subject: [PATCH 08/10] Add parent_role in targets.delegate Signed-off-by: marinamoore --- tuf/repository_tool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index f3b9c912e6..8d50216cd4 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -2412,7 +2412,8 @@ def delegate(self, rolename, public_keys, paths, threshold=1, 'keyids': keyids, 'threshold': threshold, 'terminating': terminating, - 'paths': list(relative_targetpaths.keys())} + 'paths': list(relative_targetpaths.keys()), + 'parent_role' : self._rolename} if paths: roleinfo['paths'] = paths From 76421408e9383c18959dada44a389746a8f16a96 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Tue, 4 Aug 2020 12:55:57 -0700 Subject: [PATCH 09/10] Updated delegation information in repository_tool to use different keydbs for delegations and add parent_role to roledb entries for delegations Signed-off-by: marinamoore --- tests/test_updater.py | 4 ++-- tuf/developer_tool.py | 4 ++-- tuf/repository_tool.py | 22 +++++++++++++++++----- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index 71fd319ab0..42ab8bba42 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1171,8 +1171,8 @@ def test_6_get_one_valid_targetinfo(self): repository.targets('role4').add_target(foo_package) repository.targets.load_signing_key(self.role_keys['targets']['private']) - repository.targets('role3').load_signing_key(self.role_keys['targets']['private']) - repository.targets('role4').load_signing_key(self.role_keys['targets']['private']) + repository.targets('role3').load_signing_key(self.role_keys['targets']['private'], 'targets') + repository.targets('role4').load_signing_key(self.role_keys['targets']['private'], 'targets') repository.snapshot.load_signing_key(self.role_keys['snapshot']['private']) repository.timestamp.load_signing_key(self.role_keys['timestamp']['private']) repository.writeall() diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 8a12a2a6cf..77ca259365 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -289,7 +289,7 @@ def write(self, write_partial=False): - def add_verification_key(self, key, expires=None): + def add_verification_key(self, key, expires=None, delegating_rolename='root'): """ Function as a thin wrapper call for the project._targets call @@ -322,7 +322,7 @@ def add_verification_key(self, key, expires=None): if len(self.keys) > 0: raise securesystemslib.exceptions.Error("This project already contains a key.") - super(Project, self).add_verification_key(key, expires) + super(Project, self).add_verification_key(key, expires, delegating_rolename) diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 8d50216cd4..c6fce952bf 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -682,7 +682,7 @@ def __init__(self): self._repository_name = None - def add_verification_key(self, key, expires=None): + def add_verification_key(self, key, expires=None, delegating_rolename='root'): """ Add 'key' to the role. Adding a key, which should contain only the @@ -728,6 +728,12 @@ def add_verification_key(self, key, expires=None): # 'securesystemslib.exceptions.FormatError' if any are improperly formatted. securesystemslib.formats.ANYKEY_SCHEMA.check_match(key) + # top level roles go in the default keydb, delegated roles go in the keydb + # of their parent role + repository_name = self._repository_name + if delegating_rolename != 'root': + repository_name = repository_name + ' ' + delegating_rolename + # If 'expires' is unset, choose a default expiration for 'key'. By # default, Root, Targets, Snapshot, and Timestamp keys are set to expire # 1 year, 3 months, 1 week, and 1 day from the current time, respectively. @@ -779,7 +785,7 @@ def add_verification_key(self, key, expires=None): # Keys may be shared, so do not raise an exception if 'key' has already # been loaded. try: - tuf.keydb.add_key(key, repository_name=self._repository_name) + tuf.keydb.add_key(key, repository_name=repository_name) except tuf.exceptions.KeyAlreadyExistsError: logger.warning('Adding a verification key that has already been used.') @@ -797,7 +803,7 @@ def add_verification_key(self, key, expires=None): roleinfo['keyids'].append(keyid) roleinfo['previous_keyids'] = previous_keyids - tuf.roledb.update_roleinfo(self._rolename, roleinfo, + tuf.roledb.update_roleinfo(self.rolename, roleinfo, repository_name=self._repository_name) @@ -2251,7 +2257,8 @@ def _create_delegated_target(self, rolename, keyids, threshold, paths): roleinfo = {'name': rolename, 'keyids': keyids, 'signing_keyids': [], 'threshold': threshold, 'version': 0, 'expires': expiration, 'signatures': [], 'partial_loaded': False, - 'paths': paths, 'delegations': {'keys': {}, 'roles': []}} + 'paths': paths, 'delegations': {'keys': {}, 'roles': []}, + 'parent_role' : self._parent_targets_object.rolename} # The new targets object is added as an attribute to this Targets object. new_targets_object = Targets(self._targets_directory, rolename, roleinfo, @@ -2425,8 +2432,13 @@ def delegate(self, rolename, public_keys, paths, threshold=1, del roleinfo['paths'] # Update the public keys of 'new_targets_object'. + try: + tuf.keydb.create_keydb(self._repository_name + ' ' + self._rolename) + except securesystemslib.exceptions.InvalidNameError: + # keydb already created + pass for key in public_keys: - new_targets_object.add_verification_key(key) + new_targets_object.add_verification_key(key, delegating_rolename=self._rolename) # Add the new delegation to the top-level 'targets' role object (i.e., # 'repository.targets()'). For example, 'django', which was delegated by From 7f4fd5cd18b18e7c131ff9b9aaa2332a15c2e030 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Tue, 4 Aug 2020 13:28:34 -0700 Subject: [PATCH 10/10] Add delegating_rolename to sig.py Signed-off-by: marinamoore --- tuf/sig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/sig.py b/tuf/sig.py index cd02bcfcbe..fab631e360 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -309,7 +309,7 @@ def verify(signable, role, repository_name='default', threshold=None, unique_keys = set() for keyid in good_sigs: - key = tuf.keydb.get_key(keyid, repository_name) + key = tuf.keydb.get_key(keyid, repository_name, delegating_rolename) unique_keys.add(key['keyval']['public']) return len(unique_keys) >= threshold