From fc7308af547fb8adb742a8da873c7fd0a07e4b0f Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Mon, 25 Feb 2019 17:43:08 -0500 Subject: [PATCH 01/51] Add helper funcs for roledb: get_delegation, _is_top_level_role These aid in the roledb rewrite to start to address Issue #660. Also add two minor TODOs. Signed-off-by: Sebastien Awwad --- tuf/roledb.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/tuf/roledb.py b/tuf/roledb.py index 4fabf59b1a..0771167d5d 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -113,6 +113,10 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default'): # Is 'repository_name' formatted correctly? securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) + # TODO: Confirm that none of these functions are actually changing the + # pointer, only operating within these two variables, and then remove + # all global statements for these two variables. (They're not required + # if so.) global _roledb_dict global _dirty_roles @@ -963,6 +967,124 @@ def get_delegated_rolenames(rolename, repository_name='default'): +def get_delegation( + delegated_rolename, delegating_rolename='root', repository_name='default'): + ''' + + Given a repository name and the two endpoints of a delegation, return the + delegating role's info on the delegation. + + This handles "delegations" from root to the four top-level roles as well + as delegations from any targets role (top-level or delegated) to a + delegated targets role. + + + Delegation info. This is either delegation/authorization metadata for a + top-level role or a delegated targets role. + + Top-level role delegations, provided in root metadata, are very simple, + conforming to tuf.formats.TOP_LEVEL_DELEGATION_SCHEMA. For example: + { + 'keyids': ['1234...'], + 'threshold': 1 + } + + Delegated targets role delegations (delegations from a targets role to + another targets role) are a bit more complex, conforming to + tuf.formats.DELEGATION_SCHEMA. For example: + { + "name": "role1", + "keyids": ["1234..."], + "threshold": 1 + "paths": ["file3.txt"], + "terminating": False, + } + + + tuf.exceptions.UnknownRoleEror + if the delegating_rolename is not a known role. + + tuf.exceptions.InvalidNameError + if repository_name is not a known repository + + tuf.exceptions.Error + if role delegating_rolename does not have a delegation to role + delegated_rolename + + Note that delegated_rolename does not have to be the name of a known role + in roledb; this function may be useful while roles are being loaded, and + before the entry is created for the delegated role. + + + None + ''' + + # Validate the arguments. + _check_rolename(delegating_rolename, repository_name) + tuf.formats.ROLENAME_SCHEMA.check_match(delegated_rolename) + + # Determine if the given rolename is the name of a top-level role. + top_level = _is_top_level_role(delegated_rolename) + + # Argument sanity check: top-level roles can only be delegated by root, and + # delegated targets roles cannot be delegated by root. + if top_level != (delegating_rolename == 'root'): + raise tuf.exceptions.Error( + 'Rolename ' + delegated_rolename + ' can only be delegated to by ' + 'root, not by ' + delegating_rolename) + + + if top_level: + # If we're dealing with a top-level role, the delegation information is in + # the root metadata. + + root_delegations = _roledb_dict[repository_name]['root']['roles'] + + if rolename not in root_delegations: + raise tuf.exceptions.Error( # TODO: Consider UnknownRoleError + 'Root metadata does not include delegation metadata for role ' + + rolename) + + delegation = root_delegations[rolename] + tuf.formats.TOP_LEVEL_DELEGATION_SCHEMA.check_match(delegation) + return delegation + + + else: # TODO: Make less wordy later. + # Otherwise, we're dealing with a delegated targets role, so there's no + # single source for the delegation information (authorized keys, etc.); we + # have to be told what delegating role we're interested in getting + # authorizing metadata (keys, threshold, etc.) from. + + # delegation will look like, e.g.: + # {'keyids': ['123', ...], 'threshold': 2, 'name': } + + delegations = \ + _roledb_dict[repository_name][delegating_rolename]['delegations']['roles'] + + # Note that this would be much faster with an ordered dict rather than a list + # of delegations. That would probably be slightly less understandable for + # folks perusing this reference implementation, however, and since we need + # to serialize role info to JSON, it would be a bit of a nuisance when loading + # and unloading, and complicate the metadata definition. + for delegation in delegations: + if delegation['name'] == deelgated_rolename: + tuf.formats.DELEGATION_SCHEMA.check_match(delegation) + return delegation + + raise tuf.exceptions.Error( + 'Delegation from ' + delegating_rolename + ' to ' + delegated_rolename + + ' in repository ' + repository_name + ' not found.') + + + + + + + + + + def clear_roledb(repository_name='default', clear_all=False): """ @@ -1049,6 +1171,9 @@ def _check_rolename(rolename, repository_name='default'): +# TODO: Move the ROLENAME_SCHEMA check from _check_rolename to here, and then +# strip some of the extra schema checks from functions that already use +# this function. def _validate_rolename(rolename): """ Raise securesystemslib.exceptions.InvalidNameError if 'rolename' is not @@ -1066,3 +1191,23 @@ def _validate_rolename(rolename): if rolename.startswith('/') or rolename.endswith('/'): raise securesystemslib.exceptions.InvalidNameError('Invalid rolename.' ' Cannot start or end with a "/": ' + rolename) + + + + + +def _is_top_level_role(rolename): + ''' + Simply returns True if rolename is one of the four top-level roles, and + False otherwise. + Raises tuf.exceptions.FormatError if rolename is not valid as a rolename (not + the right type, etc.). + + Note that this does not guarantee that the role exists in roledb. + ''' + tuf.formats.ROLENAME_SCHEMA.check_match(rolename) + + # TODO: We should probably integrate this list as a schema in tuf.formats. + top_level_roles = ['Root', 'Timestamp', 'Snapshot', 'Targets'] + + return rolename in top_level_roles From fd86d04c1c0c16bc389c322b70618dd5adfcbc28 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Mon, 25 Feb 2019 17:46:33 -0500 Subject: [PATCH 02/51] WIP: in roledb, remove intermediate data format; #660 - Rename and alter some schemas that really address delegations, to make that clear. - Do away with the ROLEDB_SCHEMA, an intermediate metadata format that is not necessary and which incorrectly flattens the delegation graph, and similar schemas. - Rewrite getters/setters in roledb to respect the delegation graph rather than assuming that delegated targets roles have only one delegation pointing to them (see Issue #660). - Add a variety of TODOs for later. - Clarify docstrings as a result of the above. reinterpreting metadata Signed-off-by: Sebastien Awwad --- tuf/formats.py | 64 +++++++------ tuf/roledb.py | 250 ++++++++++++++++++++++++++++++++----------------- tuf/sig.py | 2 +- 3 files changed, 199 insertions(+), 117 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index 2c5128db57..3a8bbdd297 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -100,8 +100,8 @@ # 'paths':[filepaths..]} format. # TODO: This is not a role. In further #660-related PRs, fix it, similar to # the way I did in Uptane's TUF fork. -ROLE_SCHEMA = SCHEMA.Object( - object_name = 'ROLE_SCHEMA', +DELEGATION_SCHEMA = SCHEMA.Object( + object_name = 'DELEGATION_SCHEMA', name = SCHEMA.Optional(securesystemslib.formats.ROLENAME_SCHEMA), keyids = securesystemslib.formats.KEYIDS_SCHEMA, threshold = securesystemslib.formats.THRESHOLD_SCHEMA, @@ -109,11 +109,21 @@ paths = SCHEMA.Optional(securesystemslib.formats.RELPATHS_SCHEMA), path_hash_prefixes = SCHEMA.Optional(securesystemslib.formats.PATH_HASH_PREFIXES_SCHEMA)) +# This is the data stored for each top-level role, in root metadata. +# TODO: Why is threshold schema in securesystemslib instead of here? Change +# TODO: Contemplate alternative names like AUTHENTICATION_INFO_SCHEMA. +# This is the minimal information necessary for authentication in TUF. +TOP_LEVEL_DELEGATION_SCHEMA = SCHEMA.Object( + object_name = 'TOP_LEVEL_DELEGATION_SCHEMA', + keyids = securesystemslib.formats.KEYIDS_SCHEMA, + threshold = securesystemslib.formats.THRESHOLD_SCHEMA) + +# TODO: <~> Look through where this is used and kill it or fix it. # A dict of roles where the dict keys are role names and the dict values holding # the role data/information. -ROLEDICT_SCHEMA = SCHEMA.DictOf( - key_schema = ROLENAME_SCHEMA, - value_schema = ROLE_SCHEMA) +# ROLEDICT_SCHEMA = SCHEMA.DictOf( +# key_schema = ROLENAME_SCHEMA, +# value_schema = ROLE_SCHEMA) # A dictionary of ROLEDICT, where dictionary keys can be repository names, and # dictionary values containing information for each role available on the @@ -195,16 +205,6 @@ # A list of path hash prefixes. PATH_HASH_PREFIXES_SCHEMA = SCHEMA.ListOf(PATH_HASH_PREFIX_SCHEMA) -# Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1, -# 'paths':[filepaths..]} format. -ROLE_SCHEMA = SCHEMA.Object( - object_name = 'ROLE_SCHEMA', - name = SCHEMA.Optional(ROLENAME_SCHEMA), - keyids = KEYIDS_SCHEMA, - threshold = THRESHOLD_SCHEMA, - backtrack = SCHEMA.Optional(BOOLEAN_SCHEMA), - paths = SCHEMA.Optional(RELPATHS_SCHEMA), - path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA)) # A dict of roles where the dict keys are role names and the dict values holding # the role data/information. @@ -271,7 +271,7 @@ # Like ROLEDICT_SCHEMA, except that ROLE_SCHEMA instances are stored in order. ROLELIST_SCHEMA = SCHEMA.ListOf(ROLE_SCHEMA) -# The delegated roles of a Targets role (a parent). +# The 'delegations' entry in a piece of targets role metadata. DELEGATIONS_SCHEMA = SCHEMA.Object( keys = KEYDICT_SCHEMA, roles = ROLELIST_SCHEMA) @@ -291,21 +291,23 @@ key_schema = RELPATH_SCHEMA, value_schema = CUSTOM_SCHEMA) -# TUF roledb -ROLEDB_SCHEMA = SCHEMA.Object( - object_name = 'ROLEDB_SCHEMA', - keyids = SCHEMA.Optional(KEYIDS_SCHEMA), - signing_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), - previous_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), - threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), - previous_threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), - version = SCHEMA.Optional(METADATAVERSION_SCHEMA), - expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA), - signatures = SCHEMA.Optional(securesystemslib.formats.SIGNATURES_SCHEMA), - paths = SCHEMA.Optional(SCHEMA.OneOf([RELPATHS_SCHEMA, PATH_FILEINFO_SCHEMA])), - path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA), - delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA), - partial_loaded = SCHEMA.Optional(BOOLEAN_SCHEMA)) +# TODO: <~> Kill it with fire. This is nonsensical. We use the actual +# metadata format. Maybe we add partial_loaded if we need it. +# # TUF roledb +# ROLEDB_SCHEMA = SCHEMA.Object( +# object_name = 'ROLEDB_SCHEMA', +# keyids = SCHEMA.Optional(KEYIDS_SCHEMA), +# signing_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), +# previous_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), +# threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), +# previous_threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), +# version = SCHEMA.Optional(METADATAVERSION_SCHEMA), +# expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA), +# signatures = SCHEMA.Optional(securesystemslib.formats.SIGNATURES_SCHEMA), +# paths = SCHEMA.Optional(SCHEMA.OneOf([RELPATHS_SCHEMA, PATH_FILEINFO_SCHEMA])), +# path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA), +# delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA), +# partial_loaded = SCHEMA.Optional(BOOLEAN_SCHEMA)) # A signable object. Holds the signing role and its associated signatures. SIGNABLE_SCHEMA = SCHEMA.Object( diff --git a/tuf/roledb.py b/tuf/roledb.py index 0771167d5d..507df8892b 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -132,24 +132,60 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default'): # Do not modify the contents of the 'root_metadata' argument. root_metadata = copy.deepcopy(root_metadata) - # Iterate the roles found in 'root_metadata' and add them to '_roledb_dict'. - # Duplicates are avoided. - for rolename, roleinfo in six.iteritems(root_metadata['roles']): - if rolename == 'root': - roleinfo['version'] = root_metadata['version'] - roleinfo['expires'] = root_metadata['expires'] - roleinfo['previous_keyids'] = roleinfo['keyids'] - roleinfo['previous_threshold'] = roleinfo['threshold'] + # TODO: Make sure that the schema check at the top is adequate to validate + # the contents of this metadata. Should we be finicky about optional + # args? Should I make sure there are no extra elements? Etc. - roleinfo['signatures'] = [] - roleinfo['signing_keyids'] = [] - roleinfo['partial_loaded'] = False + # Screw all the stuff below. The internal metadata format should be + # CONSISTENT throughout TUF! We use exactly what's in the metadata! + add_role('root', root_metadata, repository_name) - if rolename.startswith('targets'): - roleinfo['paths'] = {} - roleinfo['delegations'] = {'keys': {}, 'roles': []} + # TODO: See if it's necessary to add shallow entries. More likely, we + # should make fewer assumptions about these top-level roles being in + # here before they're loaded. + # # Now we add shallow entries for the other top-level roles to avoid them + # # being considered unknown roles. + # # TODO: Determine if this can be skipped. + # add_role('timestamp', {}, repository_name) + # add_role('snapshot', {}, repository_name) + # add_role('targets', {}, repository_name) - add_role(rolename, roleinfo, repository_name) + + + # # Iterate the roles found in 'root_metadata' and add them to '_roledb_dict'. + # # Duplicates are avoided. + # for rolename, roleinfo in six.iteritems(root_metadata['roles']): + # if rolename == 'root': + # # TODO: Figure out why this code only stores version and expiration in + # # roledb for root, and not for other roles? + # roleinfo['version'] = root_metadata['version'] + # roleinfo['expires'] = root_metadata['expires'] + # roleinfo['previous_keyids'] = roleinfo['keyids'] + # roleinfo['previous_threshold'] = roleinfo['threshold'] + + # #roleinfo['signatures'] = [] + # #roleinfo['signing_keyids'] = [] + # #roleinfo['partial_loaded'] = False + + # # TODO: Figure out if rolename case sensitivity is consistent across TUF. + # # TODO: Decide if we should skip these listings of non-top-level roles in + # # root metadata. + # if not _is_top_level_role(rolename.lower()): + # logger.warning( + # 'Found delegation metadata in a root role for a role that is not a ' + # 'top-level role: ' + rolename + '. Root should only be designating ' + # 'authorized signing info for top-level roles.') + + # # TODO: <~> Kill this with fire. This doesn't even make sense! Root does + # # not list delegated targets roles...... + # if rolename.startswith('targets'): + # raise Error('WTF?') + # # TODO: <~> Note that this assumes that delegated roles begin with + # # "targets". Is this still the case?? (targets/role1?) + # roleinfo['paths'] = {} + # roleinfo['delegations'] = {'keys': {}, 'roles': []} + + # add_role(rolename, roleinfo, repository_name) @@ -256,20 +292,26 @@ def add_role(rolename, roleinfo, repository_name='default'): (e.g., 'root', 'snapshot', 'timestamp'). roleinfo: - An object representing the role associated with 'rolename', conformant to - ROLEDB_SCHEMA. 'roleinfo' has the form: - {'keyids': ['34345df32093bd12...'], - 'threshold': 1, - 'signatures': ['ab23dfc32'] - 'paths': ['path/to/target1', 'path/to/target2', ...], - 'path_hash_prefixes': ['a324fcd...', ...], - 'delegations': {'keys': } - - The 'paths', 'path_hash_prefixes', and 'delegations' dict keys are - optional. - - The 'target' role has an additional 'paths' key. Its value is a list of - strings representing the path of the target file(s). + An object representing the role associated with 'rolename', conforming to + tuf.formats.ANYROLE_SCHEMA. + + For example, here's a timestamp role that could be provided as an + argument. + { + "_type": "timestamp", + "expires": "2030-01-01T00:00:00Z", + "meta": { + "snapshot.json": { + "hashes": { + "sha256": "6990b6586ed545387c6a51db62173b903a5dff46b17b1bc3fe1e6ca0d0844f2f" + }, + "length": 554, + "version": 1 + } + }, + "spec_version": "1.0", + "version": 1 + } repository_name: The name of the repository to store 'rolename'. If not supplied, @@ -298,12 +340,12 @@ def add_role(rolename, roleinfo, repository_name='default'): tuf.formats.ROLENAME_SCHEMA.check_match(rolename) # Does 'roleinfo' have the correct object format? - tuf.formats.ROLEDB_SCHEMA.check_match(roleinfo) + tuf.formats.ANYROLE_SCHEMA.check_match(roleinfo) # Is 'repository_name' correctly formatted? securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) - global _roledb_dict + global _roledb_dict # TODO: Not needed, kill. # Raises securesystemslib.exceptions.InvalidNameError. _validate_rolename(rolename) @@ -334,18 +376,26 @@ def update_roleinfo(rolename, roleinfo, mark_role_as_dirty=True, repository_name (e.g., 'root', 'snapshot', 'timestamp'). roleinfo: - An object representing the role associated with 'rolename', conformant to - ROLEDB_SCHEMA. 'roleinfo' has the form: - {'name': 'role_name', - 'keyids': ['34345df32093bd12...'], - 'threshold': 1, - 'paths': ['path/to/target1', 'path/to/target2', ...], - 'path_hash_prefixes': ['a324fcd...', ...]} - - The 'name', 'paths', and 'path_hash_prefixes' dict keys are optional. - - The 'target' role has an additional 'paths' key. Its value is a list of - strings representing the path of the target file(s). + A dictionary representing role metadata for rolename, as loaded from or + written to disk. This must conform to tuf.formats.ANYROLE_SCHEMA. + + For example, here's a timestamp role that could be provided as an + argument. + { + "_type": "timestamp", + "expires": "2030-01-01T00:00:00Z", + "meta": { + "snapshot.json": { + "hashes": { + "sha256": "6990b6586ed545387c6a51db62173b903a5dff46b17b1bc3fe1e6ca0d0844f2f" + }, + "length": 554, + "version": 1 + } + }, + "spec_version": "1.0", + "version": 1 + } mark_role_as_dirty: A boolean indicating whether the updated 'roleinfo' for 'rolename' should @@ -384,7 +434,7 @@ def update_roleinfo(rolename, roleinfo, mark_role_as_dirty=True, repository_name securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) # Does 'roleinfo' have the correct object format? - tuf.formats.ROLEDB_SCHEMA.check_match(roleinfo) + tuf.formats.ANYROLE_SCHEMA.check_match(roleinfo) # Raises securesystemslib.exceptions.InvalidNameError. _validate_rolename(rolename) @@ -687,17 +737,7 @@ def get_rolenames(repository_name='default'): def get_roleinfo(rolename, repository_name='default'): """ - Return the roleinfo of 'rolename'. - - {'keyids': ['34345df32093bd12...'], - 'threshold': 1, - 'signatures': ['ab453bdf...', ...], - 'paths': ['path/to/target1', 'path/to/target2', ...], - 'path_hash_prefixes': ['a324fcd...', ...], - 'delegations': {'keys': {}, 'roles': []}} - - The 'signatures', 'paths', 'path_hash_prefixes', and 'delegations' dict keys - are optional. + Return the roleinfo of 'rolename', conforming to tuf.formats.ANYROLE_SCHEMA rolename: @@ -743,23 +783,34 @@ def get_roleinfo(rolename, repository_name='default'): -def get_role_keyids(rolename, repository_name='default'): +def get_delegation_keyids( + rolename, repository_name='default', delegating_rolename='root'): """ - Return a list of the keyids associated with 'rolename'. Keyids are used as - identifiers for keys (e.g., rsa key). A list of keyids are associated with - each rolename. Signing a metadata file, such as 'root.json' (Root role), - involves signing or verifying the file with a list of keys identified by - keyid. + Given two roles, finds the delegation from delegating_rolename to rolename, + and returns the list of keyids authorized to sign role rolename, according + to that delegation from delegating_rolename. Searches one repository. + + If rolename is a top-level role ('targets', 'snapshot', 'root', + 'timestamp'), then the delegating role must always be 'root'. Delegated + targets roles, however, have no single authorizing role, so we must know + what targets role is doing the delegating that we care about. rolename: - An object representing the role's name, conformant to 'ROLENAME_SCHEMA' + A string representing the role's name, conformant to 'ROLENAME_SCHEMA' (e.g., 'root', 'snapshot', 'timestamp'). repository_name: - The name of the repository to get the role keyids. If not supplied, the - 'default' repository is searched. + The name of the repository whose roles we will inspect. If not supplied, + the 'default' repository is searched. + + delegating_rolename: + The name of the role delegating authority to role rolename. If this is + a top-level role, this must always be 'root'. If this is a delegated + targets role, it cannot be 'root', and should be a targets role + delegating to role rolename, along a delegation that we are interested + in. securesystemslib.exceptions.FormatError, if the arguments do not have the @@ -790,28 +841,43 @@ def get_role_keyids(rolename, repository_name='default'): global _roledb_dict global _dirty_roles - roleinfo = _roledb_dict[repository_name][rolename] + delegation = get_delegation(rolename, delegating_rolename, repository_name) - return roleinfo['keyids'] + return delegation['keyids'] -def get_role_threshold(rolename, repository_name='default'): +def get_delegation_threshold( + rolename, repository_name='default', delegating_rolename='root'): """ - Return the threshold value of the role associated with 'rolename'. + Given two roles, finds the delegation from delegating_rolename to rolename, + and returns the threshold number of keys required to sign rolename, + according to that delegation from delegating_rolename. Searches one + repository. + + If rolename is a top-level role ('targets', 'snapshot', 'root', + 'timestamp'), then the delegating role must always be 'root'. Delegated + targets roles, however, have no single authorizing role, so we must know + what targets role is doing the delegating that we care about. rolename: - An object representing the role's name, conformant to 'ROLENAME_SCHEMA' + A string representing the role's name, conformant to 'ROLENAME_SCHEMA' (e.g., 'root', 'snapshot', 'timestamp'). repository_name: - The name of the repository to get the role threshold. If not supplied, + The name of the repository whose roles we will inspect. If not supplied, the 'default' repository is searched. + delegating_rolename: + The name of the role delegating authority to role rolename. If this is + a top-level role, this must always be 'root'. If this is a delegated + targets role, it cannot be 'root', and should be a targets role + delegating to role rolename, along a delegation that we are interested + in. securesystemslib.exceptions.FormatError, if the arguments do not have the @@ -842,27 +908,42 @@ def get_role_threshold(rolename, repository_name='default'): global _roledb_dict global _dirty_roles - roleinfo = _roledb_dict[repository_name][rolename] + delegation = get_delegation(rolename, delegating_rolename, repository_name) - return roleinfo['threshold'] + return delegation['threshold'] -def get_role_paths(rolename, repository_name='default'): +def get_delegation_paths( + rolename, repository_name='default', delegating_rolename): """ - Return the paths of the role associated with 'rolename'. + Given two roles, finds the delegation from delegating_rolename to rolename, + and returns the paths delegated in that delegation. Searches one + repository. + + Only delegated targets roles are constrained to particular paths, so if + the given rolename is the name of a top-level role, an empty dictionary is + returned. + + Delegated targets roles, however, have no single authorizing role, so we + must know what targets role is doing the delegating in order to find that + delegation. rolename: - An object representing the role's name, conformant to 'ROLENAME_SCHEMA' + A string representing the role's name, conformant to 'ROLENAME_SCHEMA' (e.g., 'root', 'snapshot', 'timestamp'). repository_name: - The name of the repository to get the role paths. If not supplied, the - 'default' repository is searched. + The name of the repository whose roles we will inspect. If not supplied, + the 'default' repository is searched. + + delegating_rolename: + The name of the role delegating authority to role rolename, in the + delegation we are interested in. securesystemslib.exceptions.FormatError, if the arguments do not have the @@ -890,17 +971,16 @@ def get_role_paths(rolename, repository_name='default'): # securesystemslib.exceptions.InvalidNameError. _check_rolename(rolename, repository_name) - global _roledb_dict - global _dirty_roles - roleinfo = _roledb_dict[repository_name][rolename] + if _is_top_level_role(rolename): + # TODO: This doesn't really make a lot of sense. See if there's a reason + # to not just raise an error (which would make more sense). + return dict() - # Paths won't exist for non-target roles. - try: - return roleinfo['paths'] - except KeyError: - return dict() + delegation = get_delegation(rolename, delegating_rolename, repository_name) + + return delegation['paths'] diff --git a/tuf/sig.py b/tuf/sig.py index 3caf68b97e..fc0b309220 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -195,7 +195,7 @@ def get_signature_status(signable, role=None, repository_name='default', untrusted_sigs.append(keyid) continue - # This is an unset role, thus an unknown signature. + # This is an unset role, thus an unknown signature. # TODO: <~> THIS IS WRONG. else: unknown_sigs.append(keyid) continue From 6464f7ac1966dafc50f0e8f8cb5c643b2c63ac7b Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 2 Apr 2019 15:53:52 -0400 Subject: [PATCH 03/51] WIP, DO NOT MERGE -- quick sketch of new schemas in formats.py This is mid-development, but I'm pushing it so that Aditya can see where things are and the general shape of things. Signed-off-by: Sebastien Awwad --- tuf/formats.py | 216 ++++++++++++++++++++++++------------------------- 1 file changed, 104 insertions(+), 112 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index 3a8bbdd297..f45a7f1810 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -96,56 +96,12 @@ # A string representing a role's name. ROLENAME_SCHEMA = SCHEMA.AnyString() -# Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1, -# 'paths':[filepaths..]} format. -# TODO: This is not a role. In further #660-related PRs, fix it, similar to -# the way I did in Uptane's TUF fork. -DELEGATION_SCHEMA = SCHEMA.Object( - object_name = 'DELEGATION_SCHEMA', - name = SCHEMA.Optional(securesystemslib.formats.ROLENAME_SCHEMA), - keyids = securesystemslib.formats.KEYIDS_SCHEMA, - threshold = securesystemslib.formats.THRESHOLD_SCHEMA, - terminating = SCHEMA.Optional(securesystemslib.formats.BOOLEAN_SCHEMA), - paths = SCHEMA.Optional(securesystemslib.formats.RELPATHS_SCHEMA), - path_hash_prefixes = SCHEMA.Optional(securesystemslib.formats.PATH_HASH_PREFIXES_SCHEMA)) - -# This is the data stored for each top-level role, in root metadata. -# TODO: Why is threshold schema in securesystemslib instead of here? Change -# TODO: Contemplate alternative names like AUTHENTICATION_INFO_SCHEMA. -# This is the minimal information necessary for authentication in TUF. -TOP_LEVEL_DELEGATION_SCHEMA = SCHEMA.Object( - object_name = 'TOP_LEVEL_DELEGATION_SCHEMA', - keyids = securesystemslib.formats.KEYIDS_SCHEMA, - threshold = securesystemslib.formats.THRESHOLD_SCHEMA) - -# TODO: <~> Look through where this is used and kill it or fix it. -# A dict of roles where the dict keys are role names and the dict values holding -# the role data/information. -# ROLEDICT_SCHEMA = SCHEMA.DictOf( -# key_schema = ROLENAME_SCHEMA, -# value_schema = ROLE_SCHEMA) - -# A dictionary of ROLEDICT, where dictionary keys can be repository names, and -# dictionary values containing information for each role available on the -# repository (corresponding to the repository belonging to named repository in -# the dictionary key) -ROLEDICTDB_SCHEMA = SCHEMA.DictOf( - key_schema = securesystemslib.formats.NAME_SCHEMA, - value_schema = ROLEDICT_SCHEMA) - # Command argument list, as used by the CLI tool. # Example: {'keytype': ed25519, 'expires': 365,} COMMAND_SCHEMA = SCHEMA.DictOf( key_schema = securesystemslib.formats.NAME_SCHEMA, value_schema = SCHEMA.Any()) -# A dictionary holding version information. -VERSION_SCHEMA = SCHEMA.Object( - object_name = 'VERSION_SCHEMA', - major = SCHEMA.Integer(lo=0), - minor = SCHEMA.Integer(lo=0), - fix = SCHEMA.Integer(lo=0)) - # An integer representing the numbered version of a metadata file. # Must be 1, or greater. METADATAVERSION_SCHEMA = SCHEMA.Integer(lo=0) @@ -161,14 +117,8 @@ # Must be 1 and greater. THRESHOLD_SCHEMA = SCHEMA.Integer(lo=1) -# A hexadecimal value in '23432df87ab..' format. -HASH_SCHEMA = SCHEMA.RegularExpression(r'[a-fA-F0-9]+') - -# A hexadecimal value in '23432df87ab..' format. -HEX_SCHEMA = SCHEMA.RegularExpression(r'[a-fA-F0-9]+') - # A key identifier (e.g., a hexadecimal value identifying an RSA key). -KEYID_SCHEMA = HASH_SCHEMA +KEYID_SCHEMA = securesystemslib.formats.HASH_SCHEMA # A list of KEYID_SCHEMA. KEYIDS_SCHEMA = SCHEMA.ListOf(KEYID_SCHEMA) @@ -200,33 +150,18 @@ RELPATHS_SCHEMA = SCHEMA.ListOf(RELPATH_SCHEMA) # A path hash prefix is a hexadecimal string. -PATH_HASH_PREFIX_SCHEMA = HEX_SCHEMA +PATH_HASH_PREFIX_SCHEMA = securesystemslib.formats.HEX_SCHEMA # A list of path hash prefixes. PATH_HASH_PREFIXES_SCHEMA = SCHEMA.ListOf(PATH_HASH_PREFIX_SCHEMA) - -# A dict of roles where the dict keys are role names and the dict values holding -# the role data/information. -ROLEDICT_SCHEMA = SCHEMA.DictOf( - key_schema = ROLENAME_SCHEMA, - value_schema = ROLE_SCHEMA) - -# An integer representing length. Must be 0, or greater. -LENGTH_SCHEMA = SCHEMA.Integer(lo=0) - -# A dict in {'sha256': '23432df87ab..', 'sha512': '34324abc34df..', ...} format. -HASHDICT_SCHEMA = SCHEMA.DictOf( - key_schema = SCHEMA.AnyString(), - value_schema = HASH_SCHEMA) - # Information about target files, like file length and file hash(es). This # schema allows the storage of multiple hashes for the same file (e.g., sha256 # and sha512 may be computed for the same file and stored). FILEINFO_SCHEMA = SCHEMA.Object( object_name = 'FILEINFO_SCHEMA', - length = LENGTH_SCHEMA, - hashes = HASHDICT_SCHEMA, + length = securesystemslib.formats.LENGTH_SCHEMA, + hashes = securesystemslib.formats.HASHDICT_SCHEMA, version = SCHEMA.Optional(METADATAVERSION_SCHEMA), custom = SCHEMA.Optional(SCHEMA.Object())) @@ -246,18 +181,15 @@ # A list of TARGETINFO_SCHEMA. TARGETINFOS_SCHEMA = SCHEMA.ListOf(TARGETINFO_SCHEMA) -# A string representing a named oject. -NAME_SCHEMA = SCHEMA.AnyString() - # A dict of repository names to mirrors. REPO_NAMES_TO_MIRRORS_SCHEMA = SCHEMA.DictOf( - key_schema = NAME_SCHEMA, + key_schema = SCHEMA.AnyString(), value_schema = SCHEMA.ListOf(securesystemslib.formats.URL_SCHEMA)) # An object containing the map file's "mapping" attribute. MAPPING_SCHEMA = SCHEMA.ListOf(SCHEMA.Object( paths = RELPATHS_SCHEMA, - repositories = SCHEMA.ListOf(NAME_SCHEMA), + repositories = SCHEMA.ListOf(SCHEMA.AnyString()), terminating = BOOLEAN_SCHEMA, threshold = THRESHOLD_SCHEMA)) @@ -268,13 +200,60 @@ repositories = REPO_NAMES_TO_MIRRORS_SCHEMA, mapping = MAPPING_SCHEMA) -# Like ROLEDICT_SCHEMA, except that ROLE_SCHEMA instances are stored in order. -ROLELIST_SCHEMA = SCHEMA.ListOf(ROLE_SCHEMA) + +# SIGNING_SCHEMA is the minimal information necessary to delegate or +# authenticate in TUF. It is a list of keyids and a threshold. For example, +# the data in root metadata stored for each top-level role takes this form. +# TODO: Contemplate alternative names like AUTHENTICATION_INFO_SCHEMA. +SIGNING_SCHEMA = SCHEMA.Object( + object_name = 'SIGNING_SCHEMA', + keyids = securesystemslib.formats.KEYIDS_SCHEMA, + threshold = THRESHOLD_SCHEMA) + + +# A dict of SIGNING_SCHEMAs. The dictionary in the 'roles' field of Root +# metadata takes this form, where each top-level role has an entry listing the +# keyids and threshold Root expects of those roles. +# In this dictionary, the keys are role names and the values are SIGNING_SCHEMA +# holding keyids and threshold. +SIGNING_DICT_SCHEMA = SCHEMA.DictOf( + key_schema = ROLENAME_SCHEMA, + value_schema = SIGNING_SCHEMA) + + +# DELEGATION_SCHEMA expands on SIGNING_SCHEMA with some optional fields that +# pertain to Targets delegations. Each entry in the 'delegations' field +# DELEGATION_SCHEMA provides, at a minimum, a list of keyids and a threshold. +# This schema was previously also used for elements of the 'roles' dictionary +# in Root metadata, where keyids and threshold are provided for each top-level +# role; now, however, SIGNING_SCHEMA should be used for those. +# This schema can also be used in the delegations field of Targets metadata, +# where it is used to define a targets delegation. +# This was once "ROLE_SCHEMA", but that was a misleading name. +# A minimal example, used for a particular entry in Root's 'roles' field: +# { +# 'keyids': [, , ...], +# 'threshold': 1 +# } +# +DELEGATION_SCHEMA = SCHEMA.Object( + object_name = 'DELEGATION_SCHEMA', + name = SCHEMA.Optional(securesystemslib.formats.ROLENAME_SCHEMA), + keyids = securesystemslib.formats.KEYIDS_SCHEMA, + threshold = securesystemslib.formats.THRESHOLD_SCHEMA, + terminating = SCHEMA.Optional(securesystemslib.formats.BOOLEAN_SCHEMA), + paths = SCHEMA.Optional(securesystemslib.formats.RELPATHS_SCHEMA), + path_hash_prefixes = SCHEMA.Optional(securesystemslib.formats.PATH_HASH_PREFIXES_SCHEMA)) + # The 'delegations' entry in a piece of targets role metadata. +# The 'keys' entry contains a dictionary mapping keyid to key information. +# The 'roles' entry contains a list of DELEGATION_SCHEMA. (The specification +# requires the name 'roles', even though this is somewhat misleading as it is +# populated by delegations.) DELEGATIONS_SCHEMA = SCHEMA.Object( keys = KEYDICT_SCHEMA, - roles = ROLELIST_SCHEMA) + roles = SCHEMA.ListOf(DELEGATION_SCHEMA)) # The number of hashed bins, or the number of delegated roles. See # delegate_hashed_bins() in 'repository_tool.py' for an example. Note: @@ -291,25 +270,7 @@ key_schema = RELPATH_SCHEMA, value_schema = CUSTOM_SCHEMA) -# TODO: <~> Kill it with fire. This is nonsensical. We use the actual -# metadata format. Maybe we add partial_loaded if we need it. -# # TUF roledb -# ROLEDB_SCHEMA = SCHEMA.Object( -# object_name = 'ROLEDB_SCHEMA', -# keyids = SCHEMA.Optional(KEYIDS_SCHEMA), -# signing_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), -# previous_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), -# threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), -# previous_threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), -# version = SCHEMA.Optional(METADATAVERSION_SCHEMA), -# expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA), -# signatures = SCHEMA.Optional(securesystemslib.formats.SIGNATURES_SCHEMA), -# paths = SCHEMA.Optional(SCHEMA.OneOf([RELPATHS_SCHEMA, PATH_FILEINFO_SCHEMA])), -# path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA), -# delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA), -# partial_loaded = SCHEMA.Optional(BOOLEAN_SCHEMA)) - -# A signable object. Holds the signing role and its associated signatures. +# A signable object. Holds metadata and signatures over that metadata. SIGNABLE_SCHEMA = SCHEMA.Object( object_name = 'SIGNABLE_SCHEMA', signed = SCHEMA.Any(), @@ -395,10 +356,22 @@ expires = securesystemslib.formats.ISO8601_DATETIME_SCHEMA, mirrors = SCHEMA.ListOf(MIRROR_SCHEMA)) +# TODO: Figure out if MIRROR_SCHEMA should be removed from this list. +# (Probably) # Any of the role schemas (e.g., TIMESTAMP_SCHEMA, SNAPSHOT_SCHEMA, etc.) ANYROLE_SCHEMA = SCHEMA.OneOf([ROOT_SCHEMA, TARGETS_SCHEMA, SNAPSHOT_SCHEMA, TIMESTAMP_SCHEMA, MIRROR_SCHEMA]) +# ROLES_SCHEMA is simply a dictionary of role metadata for any of the types of +# TUF roles. +# This is used for RoleDB. RoleDB stores role metadata in memory, to manipulate +# and use before updating a client's metadata or writing new metadata. It +# takes the form of a dictionary containing a ROLES_SCHEMA for each repository +# RoleDB stores metadata from. ROLES_SCHEMA is simply a mapping from rolename +# to the role metadata for that role. +ROLES_SCHEMA = SCHEMA.DictOf(ANYROLE_SCHEMA) + +# TODO: This probably doesn't need to exist. # The format of the resulting "scp config dict" after extraction from the # push configuration file (i.e., push.cfg). In the case of a config file # utilizing the scp transfer module, it must contain the 'general' and 'scp' @@ -435,21 +408,41 @@ -def make_signable(role_schema): +def make_signable(obj): """ - Return the role metadata 'role_schema' in 'SIGNABLE_SCHEMA' format. - 'role_schema' is added to the 'signed' key, and an empty list - initialized to the 'signatures' key. The caller adds signatures - to this second field. + Returns a signable envelope dictionary around the given object. + If obj is already a signable dictionary, return that dictionary unchanged. + + # TODO: The if-it's-already-a-signable-just-return-that behavior is bad. + # Kill it. You want predictable behavior from your functions. If + # your code does something that should happen once twice, something + # is wrong and you want it to break immediately, not at some weird + # point in the future. I'm not fixing this right now because there + # are already enough things this might break and I don't want to + # complicate debugging just yet, but this has to be fixed, so TODO. + + In other words, returns a dictionary conforming to SIGNABLE_SCHEMA, of the + form: + { + 'signatures': [], + 'signed': obj + } + + The resulting dictionary can then be signed, adding signature objects + conforming to securesystemslib.formats.SIGNATURE_SCHEMA to the 'signatures' + field's list. + Note: check_signable_object_format() should be called after - make_signable() and signatures added to ensure the final - signable object has a valid format (i.e., a signable containing - a supported role metadata). + make_signable(), as well as after adding signatures, to ensure that the + final signable object has a valid format. - role_schema: - A role schema dict (e.g., 'ROOT_SCHEMA', 'SNAPSHOT_SCHEMA'). + obj: + While this was written to produce signable envelops around role metadata + dictionaries, this function supports any object (though those objects + should be serializable in order to be signed and for those signatures to + later be verified). None. @@ -458,15 +451,14 @@ def make_signable(role_schema): None. - A dict in 'SIGNABLE_SCHEMA' format. + A dictionary conforming to securesystemslib.formats.SIGNABLE_SCHEMA. """ - if not isinstance(role_schema, dict) or 'signed' not in role_schema: - return { 'signed' : role_schema, 'signatures' : [] } - - else: - return role_schema + if isinstance(object, dict) and 'signed' in object and 'signatures' in object: + # This is bad. + return object + return { 'signed' : object, 'signatures' : [] } From af271dcd284bacb8fd9d0a4dd535f6464d39360b Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Wed, 3 Apr 2019 16:29:23 -0400 Subject: [PATCH 04/51] DOC: improve formats.py module explanation/docstring Warn folks about the larger structures being near the end, make description a bit more readable, highlight matches() and check_match() funcs, emphasize that this module defines the data structures / formats used. Signed-off-by: Sebastien Awwad --- tuf/formats.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index f45a7f1810..4565065641 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -11,28 +11,35 @@ Geremy Condra Vladimir Diaz - - Refactored April 30, 2012. -vladimir.v.diaz - See LICENSE-MIT OR LICENSE for licensing information. - A central location for all format-related checking of TUF objects. - Some crypto-related formats may also be defined in securesystemslib. - Note: 'formats.py' depends heavily on 'schema.py', so the 'schema.py' - module should be read and understood before tackling this module. + A central module to define the data structures / formats used in the TUF + reference implementation, along with some functions for creating and checking + objects that conform to them. + + Because simpler components used in larger structures are defined first, + please look to the last definitions if you're looking for the metadata role + definitions. + + These definitions depend on some basic schema-defining functionality and + crypto formats from securesystemslib.schemas and securesystemslib.formats. + + 'formats.py' can be broken down into two sections: + (1) Schema definitions + (2) Functions that help produce or verify schema-conformant objects + (build_dict_conforming_to_schema, make_signable, etc.) - 'formats.py' can be broken down into two sections. (1) Schemas and object - matching. (2) Functions that help produce or verify TUF objects. - The first section deals with schemas and object matching based on format. - There are two ways of checking the format of objects. The first method - raises a 'securesystemslib.exceptions.FormatError' exception if the match - fails and the other returns a Boolean result. + Checking objects against these definitions can be done with either of two + methods: - tuf.formats..check_match(object) - tuf.formats..matches(object) + .check_match(object) + Raises FormatError if object does not match . + + .matches(object) + Returns True if object matches , else False. Example: @@ -48,11 +55,6 @@ the match fails. There are numerous variations of object checking provided by 'formats.py' and 'schema.py'. - The second section contains miscellaneous functions related to the format of - TUF objects. - Example: - - signable_object = make_signable(unsigned_object) """ # Help with Python 3 compatibility, where the print statement is a function, an From 2705b1e02d86cae51aa28b416fbe813260b63e3c Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Wed, 3 Apr 2019 16:41:10 -0400 Subject: [PATCH 05/51] Revise formats.py: version numbers, paths, minor misses - remove re-definition of rolename_schema - use securesystemslib.formats.PATH_SCHEMA for all paths, rather than using RELPATH_SCHEMA, which implies a distinction that we do not actually make, and checks we do not actually perform. - use INTEGER_NATURAL_SCHEMA for lengths and metadata versions Excludes fileinfo-related adjustments of the above, as those will follow in a separate fileinfo-specific commit. Signed-off-by: Sebastien Awwad --- tuf/formats.py | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index 4565065641..0d04eff30c 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -79,6 +79,12 @@ import six +# Version numbers for metadata and data sizes/lengths are natural integers. +INTEGER_NATURAL_SCHEMA = SCHEMA.Integer(lo=0) + +# The version of the specification with which a piece of metadata conforms is +# expressed as a string. It should conform to the typical major.minor.fix +# format version numbers commonly use, but we are not yet strict about this. SPECIFICATION_VERSION_SCHEMA = SCHEMA.AnyString() # A datetime in 'YYYY-MM-DDTHH:MM:SSZ' ISO 8601 format. The "Z" zone designator @@ -104,16 +110,9 @@ key_schema = securesystemslib.formats.NAME_SCHEMA, value_schema = SCHEMA.Any()) -# An integer representing the numbered version of a metadata file. -# Must be 1, or greater. -METADATAVERSION_SCHEMA = SCHEMA.Integer(lo=0) - # A value that is either True or False, on or off, etc. BOOLEAN_SCHEMA = SCHEMA.Boolean() -# A string representing a role's name. -ROLENAME_SCHEMA = SCHEMA.AnyString() - # A role's threshold value (i.e., the minimum number # of signatures required to sign a metadata file). # Must be 1 and greater. @@ -147,10 +146,6 @@ value_schema = KEY_SCHEMA) -# A relative file path (e.g., 'metadata/root/'). -RELPATH_SCHEMA = SCHEMA.AnyString() -RELPATHS_SCHEMA = SCHEMA.ListOf(RELPATH_SCHEMA) - # A path hash prefix is a hexadecimal string. PATH_HASH_PREFIX_SCHEMA = securesystemslib.formats.HEX_SCHEMA @@ -190,7 +185,7 @@ # An object containing the map file's "mapping" attribute. MAPPING_SCHEMA = SCHEMA.ListOf(SCHEMA.Object( - paths = RELPATHS_SCHEMA, + paths = securesystemslib.formats.PATHS_SCHEMA, repositories = SCHEMA.ListOf(SCHEMA.AnyString()), terminating = BOOLEAN_SCHEMA, threshold = THRESHOLD_SCHEMA)) @@ -244,7 +239,7 @@ keyids = securesystemslib.formats.KEYIDS_SCHEMA, threshold = securesystemslib.formats.THRESHOLD_SCHEMA, terminating = SCHEMA.Optional(securesystemslib.formats.BOOLEAN_SCHEMA), - paths = SCHEMA.Optional(securesystemslib.formats.RELPATHS_SCHEMA), + paths = SCHEMA.Optional(securesystemslib.formats.PATHS_SCHEMA), path_hash_prefixes = SCHEMA.Optional(securesystemslib.formats.PATH_HASH_PREFIXES_SCHEMA)) @@ -269,7 +264,7 @@ CUSTOM_SCHEMA = SCHEMA.Object() PATH_FILEINFO_SCHEMA = SCHEMA.DictOf( - key_schema = RELPATH_SCHEMA, + key_schema = securesystemslib.formats.PATH_SCHEMA, value_schema = CUSTOM_SCHEMA) # A signable object. Holds metadata and signatures over that metadata. @@ -283,7 +278,7 @@ object_name = 'ROOT_SCHEMA', _type = SCHEMA.String('root'), spec_version = SPECIFICATION_VERSION_SCHEMA, - version = METADATAVERSION_SCHEMA, + version = INTEGER_NATURAL_SCHEMA, consistent_snapshot = BOOLEAN_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, keys = KEYDICT_SCHEMA, @@ -294,7 +289,7 @@ object_name = 'TARGETS_SCHEMA', _type = SCHEMA.String('targets'), spec_version = SPECIFICATION_VERSION_SCHEMA, - version = METADATAVERSION_SCHEMA, + version = INTEGER_NATURAL_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, targets = FILEDICT_SCHEMA, delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA)) @@ -304,7 +299,7 @@ SNAPSHOT_SCHEMA = SCHEMA.Object( object_name = 'SNAPSHOT_SCHEMA', _type = SCHEMA.String('snapshot'), - version = securesystemslib.formats.METADATAVERSION_SCHEMA, + version = INTEGER_NATURAL_SCHEMA, expires = securesystemslib.formats.ISO8601_DATETIME_SCHEMA, spec_version = SPECIFICATION_VERSION_SCHEMA, meta = FILEINFODICT_SCHEMA) @@ -314,7 +309,7 @@ object_name = 'TIMESTAMP_SCHEMA', _type = SCHEMA.String('timestamp'), spec_version = SPECIFICATION_VERSION_SCHEMA, - version = securesystemslib.formats.METADATAVERSION_SCHEMA, + version = SCHEMA.Integer(lo=0), expires = securesystemslib.formats.ISO8601_DATETIME_SCHEMA, meta = securesystemslib.formats.FILEDICT_SCHEMA) @@ -336,9 +331,9 @@ MIRROR_SCHEMA = SCHEMA.Object( object_name = 'MIRROR_SCHEMA', url_prefix = securesystemslib.formats.URL_SCHEMA, - metadata_path = securesystemslib.formats.RELPATH_SCHEMA, - targets_path = securesystemslib.formats.RELPATH_SCHEMA, - confined_target_dirs = securesystemslib.formats.RELPATHS_SCHEMA, + metadata_path = securesystemslib.formats.PATH_SCHEMA, + targets_path = securesystemslib.formats.PATH_SCHEMA, + confined_target_dirs = securesystemslib.formats.PATHS_SCHEMA, custom = SCHEMA.Optional(SCHEMA.Object())) # A dictionary of mirrors where the dict keys hold the mirror's name and @@ -354,7 +349,7 @@ MIRRORLIST_SCHEMA = SCHEMA.Object( object_name = 'MIRRORLIST_SCHEMA', _type = SCHEMA.String('mirrors'), - version = METADATAVERSION_SCHEMA, + version = INTEGER_NATURAL_SCHEMA, expires = securesystemslib.formats.ISO8601_DATETIME_SCHEMA, mirrors = SCHEMA.ListOf(MIRROR_SCHEMA)) From 0aad781e0577f09fe6cd6deeea34b858003d182c Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Wed, 3 Apr 2019 16:46:53 -0400 Subject: [PATCH 06/51] PR revision: SIGNING_SCHEMA -> SIGNERS_SCHEMA The misleadingly-named ROLE_SCHEMA was renamed to SIGNING_SCHEMA, but I'm now making it SIGNERS_SCHEMA, which I think is clearer. I also added an example. Signed-off-by: Sebastien Awwad --- tuf/formats.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index 0d04eff30c..0e0634b83d 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -198,32 +198,39 @@ mapping = MAPPING_SCHEMA) -# SIGNING_SCHEMA is the minimal information necessary to delegate or +# SIGNERS_SCHEMA is the minimal information necessary to delegate or # authenticate in TUF. It is a list of keyids and a threshold. For example, # the data in root metadata stored for each top-level role takes this form. # TODO: Contemplate alternative names like AUTHENTICATION_INFO_SCHEMA. -SIGNING_SCHEMA = SCHEMA.Object( - object_name = 'SIGNING_SCHEMA', +# examples: +# { 'keyids': ['1234...', 'abcd...', ...], threshold: 2} +SIGNERS_SCHEMA = SCHEMA.Object( + object_name = 'SIGNERS_SCHEMA', keyids = securesystemslib.formats.KEYIDS_SCHEMA, threshold = THRESHOLD_SCHEMA) -# A dict of SIGNING_SCHEMAs. The dictionary in the 'roles' field of Root +# A dict of SIGNERS_SCHEMA dicts. The dictionary in the 'roles' field of Root # metadata takes this form, where each top-level role has an entry listing the # keyids and threshold Root expects of those roles. -# In this dictionary, the keys are role names and the values are SIGNING_SCHEMA +# In this dictionary, the keys are role names and the values are SIGNERS_SCHEMA # holding keyids and threshold. -SIGNING_DICT_SCHEMA = SCHEMA.DictOf( +# example: +# { 'root': {keyids': ['1234...', 'abcd...', ...], threshold: 2}, +# 'snapshot': {keyids': ['5656...', '9876...', ...], threshold: 1}, +# ... +# } +SIGNERS_DICT_SCHEMA = SCHEMA.DictOf( key_schema = ROLENAME_SCHEMA, - value_schema = SIGNING_SCHEMA) + value_schema = SIGNERS_SCHEMA) -# DELEGATION_SCHEMA expands on SIGNING_SCHEMA with some optional fields that +# DELEGATION_SCHEMA expands on SIGNERS_SCHEMA with some optional fields that # pertain to Targets delegations. Each entry in the 'delegations' field # DELEGATION_SCHEMA provides, at a minimum, a list of keyids and a threshold. # This schema was previously also used for elements of the 'roles' dictionary # in Root metadata, where keyids and threshold are provided for each top-level -# role; now, however, SIGNING_SCHEMA should be used for those. +# role; now, however, SIGNERS_SCHEMA should be used for those. # This schema can also be used in the delegations field of Targets metadata, # where it is used to define a targets delegation. # This was once "ROLE_SCHEMA", but that was a misleading name. @@ -282,7 +289,7 @@ consistent_snapshot = BOOLEAN_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, keys = KEYDICT_SCHEMA, - roles = ROLEDICT_SCHEMA) + roles = SIGNERS_DICT_SCHEMA) # Targets role: Indicates targets and delegates target paths to other roles. TARGETS_SCHEMA = SCHEMA.Object( From 42b14efd3e48e35cb35004605dbcff988a9aa028 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Wed, 3 Apr 2019 16:48:49 -0400 Subject: [PATCH 07/51] PR revision: minor bugfix in ROLES_SCHEMA definition Failed to include key and value definitions. Signed-off-by: Sebastien Awwad --- tuf/formats.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tuf/formats.py b/tuf/formats.py index 0e0634b83d..feb3cb7783 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -373,7 +373,9 @@ # takes the form of a dictionary containing a ROLES_SCHEMA for each repository # RoleDB stores metadata from. ROLES_SCHEMA is simply a mapping from rolename # to the role metadata for that role. -ROLES_SCHEMA = SCHEMA.DictOf(ANYROLE_SCHEMA) +ROLES_SCHEMA = SCHEMA.DictOf( + key_schema = ROLENAME_SCHEMA, + value_schema = ANYROLE_SCHEMA) # TODO: This probably doesn't need to exist. # The format of the resulting "scp config dict" after extraction from the From 49981170c0edab56e1a12e86f2d6d768cd2322de Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Wed, 3 Apr 2019 16:49:30 -0400 Subject: [PATCH 08/51] Rewrite FILEINFO schemas for clarity, and add comments In TUF, we store information about files in a variety of ways. Sometimes, versions are used, and sometimes length and hashes are required. So FILEINFO_SCHEMA will match any of these three new schemas: FILEINFO_IN_TIMESTAMP_SCHEMA, FILEINFO_IN_SNAPSHOT_SCHEMA, and FILEINFO_IN_TARGETS_SCHEMA. This should be more intuitive than the former mess, I think. I also renamed TARGETINFO to LABELED_FILEINFO_SCHEMA, with an explanation. I hope that proves more intuitive as well. Signed-off-by: Sebastien Awwad --- tuf/formats.py | 120 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index feb3cb7783..d994bcfaa5 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -93,14 +93,6 @@ # check, and an ISO8601 string should be fully verified when it is parsed. ISO8601_DATETIME_SCHEMA = SCHEMA.RegularExpression(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z') -# A dict holding the version or file information for a particular metadata -# role. The dict keys hold the relative file paths, and the dict values the -# corresponding version numbers and/or file information. -FILEINFODICT_SCHEMA = SCHEMA.DictOf( - key_schema = securesystemslib.formats.RELPATH_SCHEMA, - value_schema = SCHEMA.OneOf([securesystemslib.formats.VERSIONINFO_SCHEMA, - securesystemslib.formats.FILEINFO_SCHEMA])) - # A string representing a role's name. ROLENAME_SCHEMA = SCHEMA.AnyString() @@ -152,31 +144,89 @@ # A list of path hash prefixes. PATH_HASH_PREFIXES_SCHEMA = SCHEMA.ListOf(PATH_HASH_PREFIX_SCHEMA) -# Information about target files, like file length and file hash(es). This -# schema allows the storage of multiple hashes for the same file (e.g., sha256 -# and sha512 may be computed for the same file and stored). -FILEINFO_SCHEMA = SCHEMA.Object( - object_name = 'FILEINFO_SCHEMA', - length = securesystemslib.formats.LENGTH_SCHEMA, - hashes = securesystemslib.formats.HASHDICT_SCHEMA, - version = SCHEMA.Optional(METADATAVERSION_SCHEMA), - custom = SCHEMA.Optional(SCHEMA.Object())) -# A dict holding the information for a particular target / file. The dict keys -# hold the relative file paths, and the dict values the corresponding file -# information. -FILEDICT_SCHEMA = SCHEMA.DictOf( - key_schema = RELPATH_SCHEMA, +# FILEINFO schemas: +# In TUF, we store information about files in a variety of ways. +# Sometimes, versions are used, and sometimes length and hashes are required. +# So FILEINFO_SCHEMA will match any of these three schemas: +# FILEINFO_IN_TIMESTAMP_SCHEMA, FILEINFO_IN_SNAPSHOT_SCHEMA, +# and FILEINFO_IN_TARGETS_SCHEMA. + +# Timestamp metadata must list version, hashes, and length for the Snapshot +# metadata. +# example: +# { 'version': 7, 'length': 52, 'hashes': {'sha256': '123456...'}} +FILEINFO_IN_TIMESTAMP_SCHEMA = SCHEMA.Object( + object_name = 'FILEINFO_IN_TIMESTAMP_SCHEMA', + version = INTEGER_NATURAL_SCHEMA, + length = INTEGER_NATURAL_SCHEMA, + hashes = securesystemslib.formats.HASHDICT_SCHEMA) + +# Snapshot metadata lists only version numbers for all the Targets roles on the +# repository. (Other implementations might include hashes and length.) +# example: +# { 'version': 5 } +FILEINFO_IN_SNAPSHOT_SCHEMA = SCHEMA.Object( + object_name = 'FILEINFO_IN_SNAPSHOT_SCHEMA', + version = INTEGER_NATURAL_SCHEMA) + + +# Because Targets metadata must provide cryptographically secure information +# about the targets that must be verified, it must list hashes and length. +# It does not list version numbers, but may list additional, custom fields. +# +# Custom fields might be, for example, things like the hash to expect when an +# encrypted target file is decrypted, the file permissions recommended, authors, +# compatible part numbers, etc.). +# examples: +# {'length': 10, 'hashes': {'sha256': '123456...'}} +# { +# 'length': 10, +# 'hashes': {'sha256': '123456...'}, +# 'custom': {'arbitrary': 123, 'metadata': {1: ''}} +# } +CUSTOM_SCHEMA = SCHEMA.Object() +FILEINFO_IN_TARGETS_SCHEMA = SCHEMA.Object( + object_name= 'FILEINFO_IN_TARGETS_SCHEMA', + length = INTEGER_NATURAL_SCHEMA, + hashes = securesystemslib.formats.HASHDICT_SCHEMA, + custom = SCHEMA.Optional(CUSTOM_SCHEMA)) + +# FILEINFO_SCHEMA provides a generalization of the above FILEINFO schemas, for +# testing and modularity reasons. +FILEINFO_SCHEMA = SCHEMA.OneOf( + [FILEINFO_IN_TIMESTAMP_SCHEMA, + FILEINFO_IN_SNAPSHOT_SCHEMA, + FILEINFO_IN_TARGETS_SCHEMA]) + + +# A dictionary mapping paths or rolenames to FILEINFO_SCHEMAs. +# This is used in Timestamp, Snapshot, and Targets roles. +# +# examples: +# { 'targets': +# {'length': 10, 'hashes': {'sha256': '123456'}, 'version': 3}} +# +FILEINFO_DICT_SCHEMA = SCHEMA.DictOf( + key_schema = SCHEMA.OneOf( + [securesystemslib.formats.PATH_SCHEMA, ROLENAME_SCHEMA]), value_schema = FILEINFO_SCHEMA) -# A dict holding a target info. -TARGETINFO_SCHEMA = SCHEMA.Object( - object_name = 'TARGETINFO_SCHEMA', - filepath = RELPATH_SCHEMA, - fileinfo = FILEINFO_SCHEMA) +# LABELED_FILEINFO_SCHEMA is a filepath-labeled equivalent of +# FILEINFO_IN_TARGETS_SCHEMA. It may be of use when storing or exporting +# information about multiple targets. +# e.g. +# {'filepath': '1.tgz', +# 'fileinfo': {'length': 10, 'hashes': {'sha256': '123456'}}} +LABELED_FILEINFO_SCHEMA = SCHEMA.Object( + object_name = 'TARGELABELED_FILEINFO_SCHEMATINFO_SCHEMA', + filepath = securesystemslib.formats.PATH_SCHEMA, + fileinfo = FILEINFO_IN_TARGETS_SCHEMA) + +# A list of LABELED_FILEINFO_SCHEM objects. +LABELED_FILEINFOS_SCHEMA = SCHEMA.ListOf(LABELED_FILEINFO_SCHEMA) + -# A list of TARGETINFO_SCHEMA. -TARGETINFOS_SCHEMA = SCHEMA.ListOf(TARGETINFO_SCHEMA) # A dict of repository names to mirrors. REPO_NAMES_TO_MIRRORS_SCHEMA = SCHEMA.DictOf( @@ -259,16 +309,13 @@ keys = KEYDICT_SCHEMA, roles = SCHEMA.ListOf(DELEGATION_SCHEMA)) + # The number of hashed bins, or the number of delegated roles. See # delegate_hashed_bins() in 'repository_tool.py' for an example. Note: # Tools may require further restrictions on the number of bins, such # as requiring them to be a power of 2. NUMBINS_SCHEMA = SCHEMA.Integer(lo=1) -# The fileinfo format of targets specified in the repository and -# developer tools. The second element of this list holds custom data about the -# target, such as file permissions, author(s), last modified, etc. -CUSTOM_SCHEMA = SCHEMA.Object() PATH_FILEINFO_SCHEMA = SCHEMA.DictOf( key_schema = securesystemslib.formats.PATH_SCHEMA, @@ -298,7 +345,7 @@ spec_version = SPECIFICATION_VERSION_SCHEMA, version = INTEGER_NATURAL_SCHEMA, expires = ISO8601_DATETIME_SCHEMA, - targets = FILEDICT_SCHEMA, + targets = FILEINFO_DICT_SCHEMA, delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA)) # Snapshot role: indicates the latest versions of all metadata (except @@ -309,7 +356,7 @@ version = INTEGER_NATURAL_SCHEMA, expires = securesystemslib.formats.ISO8601_DATETIME_SCHEMA, spec_version = SPECIFICATION_VERSION_SCHEMA, - meta = FILEINFODICT_SCHEMA) + meta = FILEINFO_DICT_SCHEMA) # Timestamp role: indicates the latest version of the snapshot file. TIMESTAMP_SCHEMA = SCHEMA.Object( @@ -318,7 +365,7 @@ spec_version = SPECIFICATION_VERSION_SCHEMA, version = SCHEMA.Integer(lo=0), expires = securesystemslib.formats.ISO8601_DATETIME_SCHEMA, - meta = securesystemslib.formats.FILEDICT_SCHEMA) + meta = FILEINFO_DICT_SCHEMA) # project.cfg file: stores information about the project in a json dictionary @@ -366,6 +413,7 @@ ANYROLE_SCHEMA = SCHEMA.OneOf([ROOT_SCHEMA, TARGETS_SCHEMA, SNAPSHOT_SCHEMA, TIMESTAMP_SCHEMA, MIRROR_SCHEMA]) + # ROLES_SCHEMA is simply a dictionary of role metadata for any of the types of # TUF roles. # This is used for RoleDB. RoleDB stores role metadata in memory, to manipulate From a64fe91794e46a08838673ceeac41e13710256c6 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Wed, 3 Apr 2019 16:55:28 -0400 Subject: [PATCH 09/51] DO NOT MERGE -- adds a TODO to roledb Signed-off-by: Sebastien Awwad --- tuf/roledb.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index 507df8892b..9caee9ec24 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -917,7 +917,17 @@ def get_delegation_threshold( def get_delegation_paths( - rolename, repository_name='default', delegating_rolename): + rolename, delegating_rolename, repository_name='default'): + # TODO: <~> Deal with everything that calls get_delegation_paths such that: + # 1. the arguments are ordered correctly. (repository_name is an + # optional argument and so had to be moved from 2nd to 3rd + # argument, as another required argument was added.) + # 2. Calls are made with the delegating rolename in mind (will + # require restructuring code). + # Note that unlike with get_delegation, delegating_rolename should + # not be optional, as we will never be dealing with top-level roles + # as the delegated roles here, so we can't default to Root, and it + # doesn't make much sense to default to Targets. """ Given two roles, finds the delegation from delegating_rolename to rolename, From 0c7c64e04298c4bb7b2697f3ddb9a34e1423040b Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Mon, 15 Apr 2019 16:47:37 -0400 Subject: [PATCH 10/51] PR revision: bugfix and rename roledb._is_top_level_role It'll now be a public function used by other modules (tuf.sig), so make it public and improve the name (takes a rolename, not a role): roledb.is_top_level_rolename(). Also bugfix it to handle casing issues. Signed-off-by: Sebastien Awwad --- tuf/roledb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index 9caee9ec24..a85d89eda4 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -170,7 +170,7 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default'): # # TODO: Figure out if rolename case sensitivity is consistent across TUF. # # TODO: Decide if we should skip these listings of non-top-level roles in # # root metadata. - # if not _is_top_level_role(rolename.lower()): + # if not is_top_level_rolename(rolename.lower()): # logger.warning( # 'Found delegation metadata in a root role for a role that is not a ' # 'top-level role: ' + rolename + '. Root should only be designating ' @@ -982,7 +982,7 @@ def get_delegation_paths( _check_rolename(rolename, repository_name) - if _is_top_level_role(rolename): + if is_top_level_rolename(rolename): # TODO: This doesn't really make a lot of sense. See if there's a reason # to not just raise an error (which would make more sense). return dict() @@ -1114,7 +1114,7 @@ def get_delegation( tuf.formats.ROLENAME_SCHEMA.check_match(delegated_rolename) # Determine if the given rolename is the name of a top-level role. - top_level = _is_top_level_role(delegated_rolename) + top_level = is_top_level_rolename(delegated_rolename) # Argument sanity check: top-level roles can only be delegated by root, and # delegated targets roles cannot be delegated by root. @@ -1286,7 +1286,7 @@ def _validate_rolename(rolename): -def _is_top_level_role(rolename): +def is_top_level_rolename(rolename): ''' Simply returns True if rolename is one of the four top-level roles, and False otherwise. @@ -1298,6 +1298,6 @@ def _is_top_level_role(rolename): tuf.formats.ROLENAME_SCHEMA.check_match(rolename) # TODO: We should probably integrate this list as a schema in tuf.formats. - top_level_roles = ['Root', 'Timestamp', 'Snapshot', 'Targets'] + top_level_roles = ['root', 'timestamp', 'snapshot', 'targets'] - return rolename in top_level_roles + return rolename.lower in top_level_roles From 9a578f8535fcc6b949501690fdef0487f4b44ca2 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Mon, 15 Apr 2019 16:50:38 -0400 Subject: [PATCH 11/51] DOC: Simplify module docstring for tuf.sig It previously included information that wasn't really appropriate at this level of the code (about the project as a whole). Add short summary and list the two public functions with short explanations. Signed-off-by: Sebastien Awwad --- tuf/sig.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/tuf/sig.py b/tuf/sig.py index fc0b309220..0947e41a5a 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -17,27 +17,28 @@ See LICENSE-MIT OR LICENSE for licensing information. - Survivable key compromise is one feature of a secure update system - incorporated into TUF's design. Responsibility separation through - the use of multiple roles, multi-signature trust, and explicit and - implicit key revocation are some of the mechanisms employed towards - this goal of survivability. These mechanisms can all be seen in - play by the functions available in this module. - - The signed metadata files utilized by TUF to download target files - securely are used and represented here as the 'signable' object. - More precisely, the signature structures contained within these metadata - files are packaged into 'signable' dictionaries. This module makes it - possible to capture the states of these signatures by organizing the - keys into different categories. As keys are added and removed, the - system must securely and efficiently verify the status of these signatures. - For instance, a bunch of keys have recently expired. How many valid keys - are now available to the Snapshot role? This question can be answered by - get_signature_status(), which will return a full 'status report' of these - 'signable' dicts. This module also provides a convenient verify() function - that will determine if a role still has a sufficient number of valid keys. - If a caller needs to update the signatures of a 'signable' object, there - is also a function for that. + sig provides a higher-level signature handling interface for tuf.updater, + tuf.repository_lib, and tuf.developer_tool. Lower-level functionality used + here comes primarily from securesystemslib, tuf.roledb, and tuf.keydb. + + + + get_signature_status() + Analyzes the signatures included in given role metadata that includes + signatures, taking arguments that convey the expected keyids and + threshold for those signatures (either directly or in the form of a + rolename to look up in roledb), produces a report of the validity of the + signatures provided in the metadata indicating whether or not they + correctly sign the given metadata and whether or each signature is from + an authorized key. + + verify() + Verifies a full piece of role metadata, returning True if the given role + metadata is verified (signed by at least enough correct signatures from + authorized keys to meet the threshold expected for this metadata) and + False otherwise. It uses get_signature_status() to glean the status of + each signature. + """ # Help with Python 3 compatibility, where the print statement is a function, an From 1fcc652c71a598e4e203d597f1942688d5afff9f Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Mon, 15 Apr 2019 16:53:53 -0400 Subject: [PATCH 12/51] Refactor and correct tuf.sig for Issue #660: - get_signature_status() and verify() will expect EITHER: - keyids AND threshold - a rolename never both. A rolename can be used in place of keyids and threshold only for top-level roles, and keyids and threshold will then be drawn from currently trusted Root metadata. See comments in the code for more. This includes making rolename an optional argument. Now uses tuf.roledb.is_top_level_rolename. - renamed "role" argument to "rolename", which is more correct (since the actual role metadata is another argument...) - Pulled the elaborate argument validation and the retrieval of keyids and threshold from Root metadata into a separate function: _determine_keyids_and_threshold_to_use - perform retrieval of keyids and threshold only from Root metadata, for top-level roles (part of #660) - removed unused generate_rsa_signature - cleaned up the structure of get_signature_status() a bit Tests will break and require fixes. Signed-off-by: Sebastien Awwad --- tuf/sig.py | 425 +++++++++++++++++++++++++++-------------------------- 1 file changed, 216 insertions(+), 209 deletions(-) diff --git a/tuf/sig.py b/tuf/sig.py index 0947e41a5a..b02fe3f28f 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -67,89 +67,144 @@ iso8601_logger.disabled = True -def get_signature_status(signable, role=None, repository_name='default', +def get_signature_status(signable, rolename=None, repository_name='default', threshold=None, keyids=None): """ + # TODO: should probably be called get_status_of_signatures, plural? + - Return a dictionary representing the status of the signatures listed in - 'signable'. Given an object conformant to SIGNABLE_SCHEMA, a set of public - keys in 'tuf.keydb', a set of roles in 'tuf.roledb', and a role, - the status of these signatures can be determined. This method will iterate - the signatures in 'signable' and enumerate all the keys that are valid, - invalid, unrecognized, or unauthorized. + Given a signable role dictionary, analyzes the signatures included in it + (signable['signatures']) as signatures over the other metadata included in + it (signable['signed']). + + Returns a dictionary representing an analysis of the status of the + signatures in (see below). This is done based on the + keyids expected to sign for the role. + + If and are provided: + Uses argument to determine which keys are authorized to sign, + and returns the information along with the provided. + + If and are NOT provided and is a top-level + role: + Determines the threshold and keyids to use based on the currently trusted + Root metadata's listing for role . + + Why and are sometimes required: + # TODO: <~> Ask the reviewer in the PR if the comments below are + # important or just in the way. + Note that the reason that keyids and threshold can only automatically be + determined for top-level roles (root, snapshot, timestamp, targets) is + that top-level roles have unambiguous signature expectations: the + expected keyids and threshold come only from trusted root metadata. + Therefore, if optional args and are not provided, + the expected values can be taken from trusted Root metadata in + tuf.roledb. Delegated targets roles, on the other hand, may be the + objects of multiple different delegations from different roles that can + each have different keyid and threshold expectations, so it is not + possible to deduce these without knowing the delegating role of interest. + signable: - A dictionary containing a list of signatures and a 'signed' identifier. - signable = {'signed': 'signer', - 'signatures': [{'keyid': keyid, - 'sig': sig}]} + A dictionary with a 'signatures' entry containing a list of signatures, + and a 'signed' entry, containing role metadata. + Specifically, must conform to tuf.formats.SIGNABLE_SCHEMA, + and the 'signed' entry in must conform to + tuf.formats.ANYROLE_SCHEMA. + e.g.: + {'signatures': [ + {'keyid': '1234ef...', 'sig': 'abcd1234...'}, ... ], + 'signed': { '_type': 'root', 'version': 3, ... } + } + + rolename: (required if and are not provided) + The name of the TUF role whose metadata is provided. + If specified, this must conform to tuf.formats.ROLENAME_SCHEMA. + e.g.: 'root', 'targets', 'some_delegated_rolename', ... + This will be used to look up the required keyids and threshold to use, + from the currently trusted Root metadata's listing by role. + + threshold: (required for delegated targets roles) + If specified, this must match tuf.formats.THRESHOLD_SCHEMA. If provided + along with , this will be the information in the 'threshold' + entry of the returned dictionary. + + keyids: (required for delegated targets roles) + If specified, this must conform to tuf.formats.KEYIDS_SCHEMA. If + provided along with , this defines which keys can provide + "good" signatures over the metadata. - Conformant to tuf.formats.SIGNABLE_SCHEMA. - role: - TUF role (e.g., 'root', 'targets', 'snapshot'). + + Returns a dictionary representing the status of the signatures in + , conforming to tuf.formats.SIGNATURESTATUS_SCHEMA. The + dictionary values are lists of keyids corresponding to signatures in + , broken down under these dictionary keys: + + good_sigs: + keyids corresponding to verified signatures over the role by keys + trusted to sign over the role + + bad_sigs: + keyids corresponding to invalid signatures over the role + + unknown_sigs: + keyids (from signatures in ) from unknown keys; i.e., + keyids that had no entry in tuf.keydb. - threshold: - Rather than reference the role's threshold as set in tuf.roledb.py, use - the given 'threshold' to calculate the signature status of 'signable'. - 'threshold' is an integer value that sets the role's threshold value, or - the miminum number of signatures needed for metadata to be considered - fully signed. + untrusted_sigs: + keyids corresponding to correct signatures over the role, by known + keys (i.e. in tuf.keydb) that are nonetheless not authorized to sign + over the role. + (Authorization is based on either the argument if provided, + or, if not provided, Root metadata, as discussed below.) + + unknown_signing_scheme: + keyids corresponding to signatures that list a signing scheme that is + not supported. - keyids: - Similar to the 'threshold' argument, use the supplied list of 'keyids' - to calculate the signature status, instead of referencing the keyids - in tuf.roledb.py for 'role'. - securesystemslib.exceptions.FormatError, if 'signable' does not have the - correct format. + securesystemslib.exceptions.UnknownRoleError + if is not a known role in the repository. + + tuf.exceptions.FormatError + if , , or are not formatted + correctly, + or if is provided and not formatted correctly + or if is not provided but determined from trusted Root + metadata yet somehow formatted incorrectly there. + + securesystemslib.exceptions.FormatError + if is provided but not formatted correctly, + or if is not formatted correctly. + + tuf.exceptions.Error + if is provided but is not, or vice versa, + of if we have no way of determining the right keyids and threshold to use + in verification -- specifically, if rolename is not the name of a + top-level role and and arguments are not provided. - tuf.exceptions.UnknownRoleError, if 'role' is not recognized. None. - - - A dictionary representing the status of the signatures in 'signable'. - Conformant to tuf.formats.SIGNATURESTATUS_SCHEMA. """ - # Do the arguments have the correct format? This check will ensure that - # arguments have the appropriate number of objects and object types, and that - # all dict keys are properly named. Raise - # 'securesystemslib.exceptions.FormatError' if the check fails. + # Make sure that is correctly formatted. tuf.formats.SIGNABLE_SCHEMA.check_match(signable) - securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) - - if role is not None: - tuf.formats.ROLENAME_SCHEMA.check_match(role) - if threshold is not None: - securesystemslib.formats.THRESHOLD_SCHEMA.check_match(threshold) - - if keyids is not None: - securesystemslib.formats.KEYIDS_SCHEMA.check_match(keyids) + # This helper function will perform all other argument checks and -- if + # necessary -- look up the keyids and threshold to use from currently trusted + # Root metadata. + keyids, threshold = _determine_keyids_and_threshold_to_use( + rolename, repository_name, keyids, threshold) - # The signature status dictionary returned. + # The signature status dictionary we will return. signature_status = {} - # The fields of the signature_status dict, where each field stores keyids. A - # description of each field: - # - # good_sigs = keys confirmed to have produced 'sig' using 'signed', which are - # associated with 'role'; - # - # bad_sigs = negation of good_sigs; - # - # unknown_sigs = keys not found in the 'keydb' database; - # - # untrusted_sigs = keys that are not in the list of keyids associated with - # 'role'; - # - # unknown_signing_scheme = signing schemes specified in keys that are - # unsupported; + # The fields of the signature_status dict, where each field is a list of + # keyids. See docstring for an explanation of each. good_sigs = [] bad_sigs = [] unknown_sigs = [] @@ -166,7 +221,9 @@ def get_signature_status(signable, role=None, repository_name='default', for signature in signatures: keyid = signature['keyid'] - # Does the signature use an unrecognized key? + # Try to find the public key corresponding to the keyid (fingerprint) + # listed in the signature, so that we can actually verify the signature. + # If we can't find it, note this as an unknown key, and skip to the next. try: key = tuf.keydb.get_key(keyid, repository_name) @@ -174,7 +231,14 @@ def get_signature_status(signable, role=None, repository_name='default', unknown_sigs.append(keyid) continue - # Does the signature use an unknown/unsupported signing scheme? + # Now try verifying the signature. If the signature use an + # unknown/unsupported signing scheme and cannot be verified, note that and + # skip to the next signature. + # TODO: Make sure that verify_signature_over_metadata will actually raise + # this unsupported algorithm error appropriately. + # TODO: Note that once the next version of securesystemslib gets released, + # signed here will have to be canonicalized and encoded before it + # gets passed to verify_signature. try: valid_sig = securesystemslib.keys.verify_signature(key, signature, signed) @@ -182,32 +246,25 @@ def get_signature_status(signable, role=None, repository_name='default', unknown_signing_schemes.append(keyid) continue - # We are now dealing with either a trusted or untrusted key... - if valid_sig: - if role is not None: - - # Is this an unauthorized key? (a keyid associated with 'role') - # Note that if the role is not known, tuf.exceptions.UnknownRoleError - # is raised here. - if keyids is None: - keyids = tuf.roledb.get_role_keyids(role, repository_name) + # We know the key, we support the signing scheme, and + # verify_signature_over_metadata completed, its boolean return telling us if + # the signature is a valid signature by the key the signature mentions, + # over the data provided. + # We now ascertain whether or not this known key is one trusted to sign + # this particular metadata. - if keyid not in keyids: - untrusted_sigs.append(keyid) - continue - - # This is an unset role, thus an unknown signature. # TODO: <~> THIS IS WRONG. + if valid_sig: + # Is this an authorized key? (a keyid associated with 'role') + if keyid in keyids: + good_sigs.append(keyid) # good sig from right key else: - unknown_sigs.append(keyid) - continue - - # Identify good/authorized key. - good_sigs.append(keyid) + untrusted_sigs.append(keyid) # good sig from wrong key else: - # This is a bad signature for a trusted key. + # The signature not even valid for the key the signature says it's using. bad_sigs.append(keyid) + # Retrieve the threshold value for 'role'. Raise # securesystemslib.exceptions.UnknownRoleError if we were given an invalid # role. @@ -237,164 +294,114 @@ def get_signature_status(signable, role=None, repository_name='default', - -def verify(signable, role, repository_name='default', threshold=None, - keyids=None): +def _determine_keyids_and_threshold_to_use( + rolename, repository_name, keyids, threshold): """ - - Verify whether the authorized signatures of 'signable' meet the minimum - required by 'role'. Authorized signatures are those with valid keys - associated with 'role'. 'signable' must conform to SIGNABLE_SCHEMA - and 'role' must not equal 'None' or be less than zero. - - - signable: - A dictionary containing a list of signatures and a 'signed' identifier. - signable = {'signed':, 'signatures': [{'keyid':, 'method':, 'sig':}]} - - role: - TUF role (e.g., 'root', 'targets', 'snapshot'). - - threshold: - Rather than reference the role's threshold as set in tuf.roledb.py, use - the given 'threshold' to calculate the signature status of 'signable'. - 'threshold' is an integer value that sets the role's threshold value, or - the miminum number of signatures needed for metadata to be considered - fully signed. - - keyids: - Similar to the 'threshold' argument, use the supplied list of 'keyids' - to calculate the signature status, instead of referencing the keyids - in tuf.roledb.py for 'role'. - - - securesystemslib.exceptions.UnknownRoleError, if 'role' is not recognized. - - securesystemslib.exceptions.FormatError, if 'signable' is not formatted - correctly. - - securesystemslib.exceptions.Error, if an invalid threshold is encountered. - - - tuf.sig.get_signature_status() called. Any exceptions thrown by - get_signature_status() will be caught here and re-raised. - - - Boolean. True if the number of good signatures >= the role's threshold, - False otherwise. + Helper function for get_signature_status. Tests all the various argument + constraints for get_signature_status and looks up the keyids and threshold if + necessary. See docstring for get_signature_status. """ - tuf.formats.SIGNABLE_SCHEMA.check_match(signable) - tuf.formats.ROLENAME_SCHEMA.check_match(role) securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) - # Retrieve the signature status. tuf.sig.get_signature_status() raises: - # securesystemslib.exceptions.UnknownRoleError - # securesystemslib.exceptions.FormatError. 'threshold' and 'keyids' are also - # validated. - status = get_signature_status(signable, role, repository_name, threshold, keyids) + # Sanity check for argument pairs: + if (keyids is None) != (threshold is None) or \ + (keyids is None) == (rolename is None): + raise tuf.exceptions.Error( + 'This function must be called using either the "rolename" argument OR ' + 'using the "keyids" and "threshold" arguments. One set or the other ' + 'must be provided, and not both sets. ' + '(keyids provided? ' + str(keyids is not None) + + '; threshold provided? ' + str(threshold is not None) + + '; rolename provided? ' + str(rolename is not None) + ')') - # Retrieve the role's threshold and the authorized keys of 'status' - threshold = status['threshold'] - good_sigs = status['good_sigs'] - - # Does 'status' have the required threshold of signatures? - # First check for invalid threshold values before returning result. - # Note: get_signature_status() is expected to verify that 'threshold' is - # not None or <= 0. - if threshold is None or threshold <= 0: #pragma: no cover - raise securesystemslib.exceptions.Error("Invalid threshold: " + repr(threshold)) + if keyids is not None: - return len(good_sigs) >= threshold + # DEBUG ONLY: REMOVE AFTER TESTING: + assert threshold is not None, 'Not possible; mistake in this function!' + assert rolename is None, 'Not possible: mistake in this function!' + securesystemslib.formats.KEYIDS_SCHEMA.check_match(keyids) + tuf.formats.THRESHOLD_SCHEMA.check_match(threshold) + # We were given keyids and threshold and their formats check out. + return keyids, threshold + # Otherwise, we weren't provided keyids and threshold, so figure them out if + # possible. + # DEBUG ONLY: REMOVE AFTER TESTING: + assert threshold is None and keyids is None, 'Not possible; mistake in this function!' + assert rolename is not None, 'Not possible; mistake in this function!' -def may_need_new_keys(signature_status): - """ - - Return true iff downloading a new set of keys might tip this - signature status over to valid. This is determined by checking - if either the number of unknown or untrused keys is > 0. + tuf.formats.ROLENAME_SCHEMA.check_match(rolename) + if not roledb.is_top_level_rolename(rolename): + raise tuf.exceptions.Error( + 'Cannot automatically determine the keyids and threshold expected of ' + 'a delegated targets role ("' + rolename + '"). The rolename ' + 'argument is sufficient only for roles listed by Root. The name of ' + 'a delegated role need never be provided as argument.') - - signature_status: - The dictionary returned by tuf.sig.get_signature_status(). + keyids = tuf.roledb.get_role_keyids(rolename, repository_name) + threshold = tuf.roledb.get_role_threshold( + rolename, repository_name=repository_name) - - securesystemslib.exceptions.FormatError, if 'signature_status does not have - the correct format. + # Check the results. + # TODO: Determine if this is overkill. It's probably checked already + # before it is returned. + securesystemslib.formats.KEYIDS_SCHEMA.check_match(keyids) + tuf.formats.THRESHOLD_SCHEMA.check_match(threshold) - - None. - - Boolean. - """ - # Does 'signature_status' have the correct format? - # This check will ensure 'signature_status' has the appropriate number - # of objects and object types, and that all dict keys are properly named. - # Raise 'securesystemslib.exceptions.FormatError' if the check fails. - securesystemslib.formats.SIGNATURESTATUS_SCHEMA.check_match(signature_status) - unknown = signature_status['unknown_sigs'] - untrusted = signature_status['untrusted_sigs'] - return len(unknown) or len(untrusted) - - - - - -def generate_rsa_signature(signed, rsakey_dict): +def verify(signable, rolename=None, repository_name='default', threshold=None, + keyids=None): """ - Generate a new signature dict presumably to be added to the 'signatures' - field of 'signable'. The 'signable' dict is of the form: + Verify whether the signatures in meet the requirements. - {'signed': 'signer', - 'signatures': [{'keyid': keyid, - 'method': 'evp', - 'sig': sig}]} + Returns True if there are at least a threshold of valid signatures over the + 'signed' component of by distinct keys with keyids in a list + of authorized keyids, else returns False. - The 'signed' argument is needed here for the signing process. - The 'rsakey_dict' argument is used to generate 'keyid', 'method', and 'sig'. + The list of authorized keys and the threshold of signatures required may + be passed in as and . Alternatively, if they are not + provided, the keyids and threshold will be determined based on the + currently trusted Root metadata's listing for role , but that + only works if the role being verified is a top-level role. - The caller should ensure the returned signature is not already in - 'signable'. - - signed: - The data used by 'securesystemslib.keys.create_signature()' to generate - signatures. It is stored in the 'signed' field of 'signable'. - - rsakey_dict: - The RSA key, a 'securesystemslib.formats.RSAKEY_SCHEMA' dictionary. - Used here to produce 'keyid', 'method', and 'sig'. + This wraps get_signature_status(), takes the same arguments, and raises + the same errors, so please see the docstring for get_signature_status(). - - securesystemslib.exceptions.FormatError, if 'rsakey_dict' does not have the - correct format. - TypeError, if a private key is not defined for 'rsakey_dict'. + + Boolean. True if the number of good signatures >= the role's threshold, + False otherwise. None. - - - Signature dictionary conformant to securesystemslib.formats.SIGNATURE_SCHEMA. - Has the form: - {'keyid': keyid, 'method': 'evp', 'sig': sig} """ - # We need 'signed' in canonical JSON format to generate - # the 'method' and 'sig' fields of the signature. - signed = securesystemslib.formats.encode_canonical(signed) + # Note that get_signature_status() checks all arguments, so argument + # checking here is skipped. + + # Retrieve the status of signatures included in argument . + # tuf.sig.get_signature_status() raises: + # securesystemslib.exceptions.UnknownRoleError, + # tuf.exceptions.FormatError, and + # securesystemslib.exceptions.FormatError + # tuf.exceptions.Error if the role is a delegated targets role but keyids and + # threshold are not provided. + status = get_signature_status( + signable, rolename, repository_name, threshold, keyids) + + # Retrieve the role's threshold and the authorized keys of 'status' + threshold = status['threshold'] + good_sigs = status['good_sigs'] - # Generate the RSA signature. - # Raises securesystemslib.exceptions.FormatError and TypeError. - signature = securesystemslib.keys.create_signature(rsakey_dict, signed) + # Does 'status' have the required threshold of signatures? - return signature + return len(good_sigs) >= threshold From cd3d470410a5f784238911cfc89d5a25cac83915 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 10:33:49 -0400 Subject: [PATCH 13/51] PR revision: remove leftover code in get_signature_status: The keyids and threshold retrieval are already performed above now, so this lingering threshold retrieval is no longer needed. Move the comment about errors it would raise to where that actually would happen now (and refine comment given new functionality). Signed-off-by: Sebastien Awwad --- tuf/sig.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/tuf/sig.py b/tuf/sig.py index b02fe3f28f..79fb0097ca 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -254,7 +254,7 @@ def get_signature_status(signable, rolename=None, repository_name='default', # this particular metadata. if valid_sig: - # Is this an authorized key? (a keyid associated with 'role') + # Is this an authorized key? (a keyid associated with ) if keyid in keyids: good_sigs.append(keyid) # good sig from right key else: @@ -265,22 +265,6 @@ def get_signature_status(signable, rolename=None, repository_name='default', bad_sigs.append(keyid) - # Retrieve the threshold value for 'role'. Raise - # securesystemslib.exceptions.UnknownRoleError if we were given an invalid - # role. - if role is not None: - if threshold is None: - # Note that if the role is not known, tuf.exceptions.UnknownRoleError is - # raised here. - threshold = tuf.roledb.get_role_threshold( - role, repository_name=repository_name) - - else: - logger.debug('Not using roledb.py\'s threshold for ' + repr(role)) - - else: - threshold = 0 - # Build the signature_status dict. signature_status['threshold'] = threshold signature_status['good_sigs'] = good_sigs @@ -342,6 +326,13 @@ def _determine_keyids_and_threshold_to_use( 'argument is sufficient only for roles listed by Root. The name of ' 'a delegated role need never be provided as argument.') + + # Pull the keyids and threshold expected for this top-level role from trusted + # Root metadata. Note that if the rolename is not known, + # tuf.exceptions.UnknownRoleError is raised here. Argument checks above + # ensure that `rolename` is the name of a top-level role, so this should only + # be possible if there is no Root metadata loaded somehow, or if that Root + # metadata is missing a listing for a top-level role for some reason.... keyids = tuf.roledb.get_role_keyids(rolename, repository_name) threshold = tuf.roledb.get_role_threshold( rolename, repository_name=repository_name) From 43750ba0646cbbd11de9902f9b8ef912d3ef43c1 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 12:17:51 -0400 Subject: [PATCH 14/51] Remove tests for deleted functions in tuf.sig tuf.sig.generate_rsa_signature and tuf.sig.may_need_new_keys were not necessary and were deleted. This commit removes their tests. Signed-off-by: Sebastien Awwad --- tests/test_sig.py | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/tests/test_sig.py b/tests/test_sig.py index de671a6468..48ba5d5584 100755 --- a/tests/test_sig.py +++ b/tests/test_sig.py @@ -406,53 +406,6 @@ def test_verify_unrecognized_sig(self): - def test_generate_rsa_signature(self): - signable = {'signed' : 'test', 'signatures' : []} - - signable['signatures'].append(securesystemslib.keys.create_signature( - KEYS[0], signable['signed'])) - - self.assertEqual(1, len(signable['signatures'])) - signature = signable['signatures'][0] - self.assertEqual(KEYS[0]['keyid'], signature['keyid']) - - returned_signature = tuf.sig.generate_rsa_signature(signable['signed'], KEYS[0]) - self.assertTrue(securesystemslib.formats.SIGNATURE_SCHEMA.matches(returned_signature)) - - signable['signatures'].append(securesystemslib.keys.create_signature( - KEYS[1], signable['signed'])) - - self.assertEqual(2, len(signable['signatures'])) - signature = signable['signatures'][1] - self.assertEqual(KEYS[1]['keyid'], signature['keyid']) - - - - def test_may_need_new_keys(self): - # One untrusted key in 'signable'. - signable = {'signed' : 'test', 'signatures' : []} - - signable['signatures'].append(securesystemslib.keys.create_signature( - KEYS[0], signable['signed'])) - - tuf.keydb.add_key(KEYS[1]) - threshold = 1 - - roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, keyids=[KEYS[1]['keyid']], threshold=threshold) - - tuf.roledb.add_role('Root', roleinfo) - - sig_status = tuf.sig.get_signature_status(signable, 'Root') - - self.assertTrue(tuf.sig.may_need_new_keys(sig_status)) - - - # Done. Let's remove the added key(s) from the key database. - tuf.keydb.remove_key(KEYS[1]['keyid']) - - # Remove the roles. - tuf.roledb.remove_role('Root') def test_signable_has_invalid_format(self): From cdae6b30e0211c7f5eaf6674bfc24666d5f33257 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 12:20:20 -0400 Subject: [PATCH 15/51] minor: DOC: clarify comment in test_sig Explain the test conditions. Signed-off-by: Sebastien Awwad --- tests/test_sig.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_sig.py b/tests/test_sig.py index 48ba5d5584..db1ed239ef 100755 --- a/tests/test_sig.py +++ b/tests/test_sig.py @@ -191,7 +191,10 @@ def test_get_signature_status_single_key(self): self.assertTrue(tuf.sig.verify(signable, 'Root')) - # Test for an unknown signature when 'role' is left unspecified. + # If get_signature_status is not provided authorized keyids and threshold, + # and is also not provided a role to use to determine what keyids and + # threshold are authorized, then we expect any good signature to come back + # as untrustworthy, and any bad signature to come back as a bad signature. sig_status = tuf.sig.get_signature_status(signable) self.assertEqual(0, sig_status['threshold']) From e6660cb5284adef868d420955dccb8b42388718f Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 16 Apr 2019 12:21:03 -0400 Subject: [PATCH 16/51] DO NOT MERGE: add TODOs to test_sig for after roledb changes Signed-off-by: Sebastien Awwad --- tests/test_sig.py | 67 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/tests/test_sig.py b/tests/test_sig.py index db1ed239ef..d9c781e534 100755 --- a/tests/test_sig.py +++ b/tests/test_sig.py @@ -109,8 +109,14 @@ def test_get_signature_status_bad_sig(self): tuf.keydb.add_key(KEYS[0]) threshold = 1 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, keyids=[KEYS[0]['keyid']], threshold=threshold) + tuf.formats.SIGNERS_SCHEMA, + keyids=[KEYS[0]['keyid']], threshold=threshold) tuf.roledb.add_role('Root', roleinfo) @@ -142,8 +148,13 @@ def test_get_signature_status_unknown_signing_scheme(self): tuf.keydb.add_key(KEYS[0]) threshold = 1 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, keyids=[KEYS[0]['keyid']], threshold=threshold) + tuf.formats.SIGNERS_SCHEMA, keyids=[KEYS[0]['keyid']], threshold=threshold) tuf.roledb.add_role('root', roleinfo) @@ -174,8 +185,14 @@ def test_get_signature_status_single_key(self): threshold = 1 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, keyids=[KEYS[0]['keyid']], threshold=threshold) + tuf.formats.SIGNERS_SCHEMA, + keyids=[KEYS[0]['keyid']], threshold=threshold) tuf.roledb.add_role('Root', roleinfo) tuf.keydb.add_key(KEYS[0]) @@ -200,8 +217,10 @@ def test_get_signature_status_single_key(self): self.assertEqual(0, sig_status['threshold']) self.assertEqual([], sig_status['good_sigs']) self.assertEqual([], sig_status['bad_sigs']) - self.assertEqual([KEYS[0]['keyid']], sig_status['unknown_sigs']) - self.assertEqual([], sig_status['untrusted_sigs']) + # TODO: <~> Add this comment to the commit message instead: + # Correct bad expectation: if role is not provided, then all signatures + self.assertEqual([], sig_status['unknown_sigs']) + self.assertEqual([KEYS[0]['keyid']], sig_status['untrusted_sigs']) self.assertEqual([], sig_status['unknown_signing_schemes']) # Done. Let's remove the added key(s) from the key database. @@ -219,8 +238,13 @@ def test_get_signature_status_below_threshold(self): tuf.keydb.add_key(KEYS[0]) threshold = 2 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, + tuf.formats.SIGNERS_SCHEMA, keyids=[KEYS[0]['keyid'], KEYS[2]['keyid']], threshold=threshold) @@ -257,8 +281,13 @@ def test_get_signature_status_below_threshold_unrecognized_sigs(self): tuf.keydb.add_key(KEYS[1]) threshold = 2 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, + tuf.formats.SIGNERS_SCHEMA, keyids=[KEYS[0]['keyid'], KEYS[1]['keyid']], threshold=threshold) @@ -297,15 +326,20 @@ def test_get_signature_status_below_threshold_unauthorized_sigs(self): tuf.keydb.add_key(KEYS[1]) threshold = 2 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, + tuf.formats.SIGNERS_SCHEMA, keyids=[KEYS[0]['keyid'], KEYS[2]['keyid']], threshold=threshold) tuf.roledb.add_role('Root', roleinfo) roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, + tuf.formats.SIGNERS_SCHEMA, keyids=[KEYS[1]['keyid'], KEYS[2]['keyid']], threshold=threshold) @@ -362,8 +396,14 @@ def test_verify_single_key(self): tuf.keydb.add_key(KEYS[0]) threshold = 1 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, keyids=[KEYS[0]['keyid']], threshold=threshold) + tuf.formats.SIGNERS_SCHEMA, + keyids=[KEYS[0]['keyid']], threshold=threshold) tuf.roledb.add_role('Root', roleinfo) @@ -391,8 +431,13 @@ def test_verify_unrecognized_sig(self): tuf.keydb.add_key(KEYS[1]) threshold = 2 + # TODO: roledb no longer stores metadata in this format. Once you're all + # done with roledb, this will need to load a full sample Root + # metadata file here in order to populate roledb with the Root info, + # not just the keyid & threshold info. + roleinfo = tuf.formats.build_dict_conforming_to_schema( - tuf.formats.ROLE_SCHEMA, + tuf.formats.SIGNERS_SCHEMA, keyids=[KEYS[0]['keyid'], KEYS[1]['keyid']], threshold=threshold) From fe8530f5283cf44dadf0e4b8dc5073376ff972fa Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Fri, 5 Apr 2019 14:56:04 -0400 Subject: [PATCH 17/51] Update metadata for roleinfo to match expected - #660 --- tests/test_roledb.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index ebb33a20fe..da40979dd0 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -623,7 +623,18 @@ def test_create_roledb_from_root_metadata(self): def test_update_roleinfo(self): rolename = 'targets' - roleinfo = {'keyids': ['123'], 'threshold': 1} + roleinfo = {'_type': 'targets', + 'spec_version': '1.0', + 'targets': { + 'some_target': { + 'length': 0, + 'hashes': { + 'ab23143': 'abcd12134213abf' + } + } + }, + 'version': 1, + 'expires': '2030-01-01T00:00:00Z'} tuf.roledb.add_role(rolename, roleinfo) # Test normal case. From 7e5e90aba12e4d7796faba2d2b5eec25395af290 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Sat, 6 Apr 2019 13:00:50 -0400 Subject: [PATCH 18/51] Modify metadata to match new expected schemas --- tests/test_roledb.py | 124 ++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 65 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index da40979dd0..7144a079d1 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -39,6 +39,7 @@ import securesystemslib import securesystemslib.keys +import os logger = logging.getLogger('tuf.test_roledb') @@ -53,6 +54,7 @@ class TestRoledb(unittest.TestCase): def setUp(self): tuf.roledb.clear_roledb(clear_all=True) + self.test_data_path = os.path.join(os.getcwd(), "tests", "repository_data", "repository", "metadata") @@ -107,20 +109,23 @@ def test_clear_roledb(self): # Test for an empty roledb, a length of 1 after adding a key, and finally # an empty roledb after calling 'clear_roledb()'. self.assertEqual(0, len(tuf.roledb._roledb_dict['default'])) - tuf.roledb._roledb_dict['default']['Root'] = {'keyids': ['123'], 'threshold': 1} + tuf.roledb._roledb_dict['default']['Root'] =\ + securesystemslib.util.load_json_file(os.path.join(self.test_data_path, + "root.json"))['signed'] self.assertEqual(1, len(tuf.roledb._roledb_dict['default'])) tuf.roledb.clear_roledb() self.assertEqual(0, len(tuf.roledb._roledb_dict['default'])) # Verify that the roledb can be cleared for a non-default repository. rolename = 'targets' - roleinfo = {'keyids': ['123'], 'threshold': 1} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] repository_name = 'example_repository' self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.clear_roledb, repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(roleinfo['keyids'], tuf.roledb.get_role_keyids(rolename, repository_name)) + # TODO remove keyid check + # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) tuf.roledb.clear_roledb(repository_name) self.assertFalse(tuf.roledb.role_exists(rolename, repository_name)) @@ -138,7 +143,7 @@ def test_add_role(self): # Test conditions where the arguments are valid. self.assertEqual(0, len(tuf.roledb._roledb_dict['default'])) rolename = 'targets' - roleinfo = {'keyids': ['123'], 'threshold': 1} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] rolename2 = 'role1' self.assertEqual(None, tuf.roledb.add_role(rolename, roleinfo)) self.assertEqual(1, len(tuf.roledb._roledb_dict['default'])) @@ -152,8 +157,9 @@ def test_add_role(self): repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(roleinfo['keyids'], tuf.roledb.get_role_keyids(rolename, - repository_name)) + # TODO remove keyid check + # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, + # repository_name)) # Reset the roledb so that subsequent tests have access to a default # roledb. @@ -190,7 +196,7 @@ def test_add_role(self): def test_role_exists(self): # Test conditions where the arguments are valid. rolename = 'targets' - roleinfo = {'keyids': ['123'], 'threshold': 1} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] rolename2 = 'role1' self.assertEqual(False, tuf.roledb.role_exists(rolename)) @@ -233,11 +239,8 @@ def test_remove_role(self): rolename = 'targets' rolename2 = 'release' rolename3 = 'django' - roleinfo = {'keyids': ['123'], 'threshold': 1} - roleinfo2 = {'keyids': ['123'], 'threshold': 1, 'delegations': - {'roles': [{'name': 'django', 'keyids': ['456'], 'threshold': 1}], - 'keys': {'456': {'keytype': 'rsa', 'keyval': {'public': '456'}}, - }}} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) @@ -252,7 +255,8 @@ def test_remove_role(self): tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(roleinfo['keyids'], tuf.roledb.get_role_keyids(rolename, repository_name)) + # TODO remove keyid check + # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) self.assertEqual(None, tuf.roledb.remove_role(rolename, repository_name)) # Verify that a role cannot be removed from a non-existent repository name. @@ -281,7 +285,7 @@ def test_get_rolenames(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = {'keyids': ['123'], 'threshold': 1} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] self.assertEqual([], tuf.roledb.get_rolenames()) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo) @@ -312,8 +316,8 @@ def test_get_role_info(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = {'keyids': ['123'], 'threshold': 1} - roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_roleinfo, rolename) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) @@ -348,32 +352,32 @@ def test_get_role_info(self): - def test_get_role_keyids(self): + def test_get_delegation_keyids(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' roleinfo = {'keyids': ['123'], 'threshold': 1} roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} - self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_role_keyids, rolename) + self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_keyids, rolename) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) - self.assertEqual(['123'], tuf.roledb.get_role_keyids(rolename)) + self.assertEqual(['123'], tuf.roledb.get_delegation_keyids(rolename)) self.assertEqual(set(['456', '789']), - set(tuf.roledb.get_role_keyids(rolename2))) + set(tuf.roledb.get_delegation_keyids(rolename2))) # Verify that the role keyids can be retrieved for a role in a non-default # repository. repository_name = 'example_repository' - self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_role_keyids, + self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_keyids, rolename, repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(['123'], tuf.roledb.get_role_keyids(rolename, repository_name)) + self.assertEqual(['123'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) # Verify that rolekeyids cannot be retrieved from a non-existent repository # name. - self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_role_keyids, rolename, + self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_keyids, rolename, 'non-existent') # Reset the roledb so that subsequent tests have access to the original, @@ -382,36 +386,36 @@ def test_get_role_keyids(self): # Test conditions where the arguments are improperly formatted, contain # invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_role_keyids) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_role_keyids, rolename, 123) + self._test_rolename(tuf.roledb.get_delegation_keyids) + self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_keyids, rolename, 123) - def test_get_role_threshold(self): + def test_get_delegation_threshold(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' roleinfo = {'keyids': ['123'], 'threshold': 1} roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} - self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_role_threshold, rolename) + self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_threshold, rolename) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) - self.assertEqual(1, tuf.roledb.get_role_threshold(rolename)) - self.assertEqual(2, tuf.roledb.get_role_threshold(rolename2)) + self.assertEqual(1, tuf.roledb.get_delegation_threshold(rolename)) + self.assertEqual(2, tuf.roledb.get_delegation_threshold(rolename2)) # Verify that the threshold can be retrieved for a role in a non-default # repository. repository_name = 'example_repository' - self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_role_threshold, + self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_threshold, rolename, repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(roleinfo['threshold'], tuf.roledb.get_role_threshold(rolename, repository_name)) + self.assertEqual(roleinfo['threshold'], tuf.roledb.get_delegation_threshold(rolename, repository_name)) # Verify that a role's threshold cannot be retrieved from a non-existent # repository name. - self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_role_threshold, + self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_threshold, rolename, 'non-existent') # Reset the roledb so that subsequent tests have access to the original, @@ -420,33 +424,33 @@ def test_get_role_threshold(self): # Test conditions where the arguments are improperly formatted, # contain invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_role_threshold) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_role_threshold, rolename, 123) + self._test_rolename(tuf.roledb.get_delegation_threshold) + self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_threshold, rolename, 123) - def test_get_role_paths(self): + def test_get_delegation_paths(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' roleinfo = {'keyids': ['123'], 'threshold': 1} paths = ['a/b', 'c/d'] roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2, 'paths': paths} - self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_role_paths, rolename) + self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_paths, rolename) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) - self.assertEqual({}, tuf.roledb.get_role_paths(rolename)) - self.assertEqual(paths, tuf.roledb.get_role_paths(rolename2)) + self.assertEqual({}, tuf.roledb.get_delegation_paths(rolename)) + self.assertEqual(paths, tuf.roledb.get_delegation_paths(rolename2)) # Verify that role paths can be queried for roles in non-default # repositories. repository_name = 'example_repository' - self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_role_paths, + self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_paths, rolename, repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename2, roleinfo2, repository_name) - self.assertEqual(roleinfo2['paths'], tuf.roledb.get_role_paths(rolename2, + self.assertEqual(roleinfo2['paths'], tuf.roledb.get_delegation_paths(rolename2, repository_name)) # Reset the roledb so that subsequent roles have access to the original, @@ -455,8 +459,8 @@ def test_get_role_paths(self): # Test conditions where the arguments are improperly formatted, # contain invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_role_paths) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_role_paths, rolename, 123) + self._test_rolename(tuf.roledb.get_delegation_paths) + self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_paths, rolename, 123) @@ -560,16 +564,16 @@ def test_create_roledb_from_root_metadata(self): tuf.roledb.create_roledb_from_root_metadata(root_metadata)) # Ensure 'Root' and 'Targets' were added to the role database. - self.assertEqual([keyid], tuf.roledb.get_role_keyids('root')) - self.assertEqual([keyid2], tuf.roledb.get_role_keyids('targets')) + self.assertEqual([keyid], tuf.roledb.get_delegation_keyids('root')) + self.assertEqual([keyid2], tuf.roledb.get_delegation_keyids('targets')) # Test that a roledb is created for a non-default repository. repository_name = 'example_repository' self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.clear_roledb, repository_name) tuf.roledb.create_roledb_from_root_metadata(root_metadata, repository_name) - self.assertEqual([keyid], tuf.roledb.get_role_keyids('root', repository_name)) - self.assertEqual([keyid2], tuf.roledb.get_role_keyids('targets', repository_name)) + self.assertEqual([keyid], tuf.roledb.get_delegation_keyids('root', repository_name)) + self.assertEqual([keyid2], tuf.roledb.get_delegation_keyids('targets', repository_name)) # Remove the example repository added to the roledb so that subsequent # tests have access to an original, default roledb. @@ -623,18 +627,7 @@ def test_create_roledb_from_root_metadata(self): def test_update_roleinfo(self): rolename = 'targets' - roleinfo = {'_type': 'targets', - 'spec_version': '1.0', - 'targets': { - 'some_target': { - 'length': 0, - 'hashes': { - 'ab23143': 'abcd12134213abf' - } - } - }, - 'version': 1, - 'expires': '2030-01-01T00:00:00Z'} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo) # Test normal case. @@ -648,7 +641,8 @@ def test_update_roleinfo(self): tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) tuf.roledb.update_roleinfo(rolename, roleinfo, mark_role_as_dirty, repository_name) - self.assertEqual(roleinfo['keyids'], tuf.roledb.get_role_keyids(rolename, repository_name)) + # TODO remove keyid check + # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) # Reset the roledb so that subsequent tests can access the default roledb. tuf.roledb.remove_roledb(repository_name) @@ -678,9 +672,9 @@ def test_update_roleinfo(self): def test_get_dirty_roles(self): # Verify that the dirty roles of a role are returned. rolename = 'targets' - roleinfo1 = {'keyids': ['123'], 'threshold': 1} + roleinfo1 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo1) - roleinfo2 = {'keyids': ['123'], 'threshold': 2} + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] mark_role_as_dirty = True tuf.roledb.update_roleinfo(rolename, roleinfo2, mark_role_as_dirty) # Note: The 'default' repository is searched if the repository name is @@ -710,10 +704,10 @@ def test_get_dirty_roles(self): def test_mark_dirty(self): # Add a dirty role to roledb. rolename = 'targets' - roleinfo1 = {'keyids': ['123'], 'threshold': 1} + roleinfo1 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo1) rolename2 = 'dirty_role' - roleinfo2 = {'keyids': ['123'], 'threshold': 2} + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] mark_role_as_dirty = True tuf.roledb.update_roleinfo(rolename, roleinfo1, mark_role_as_dirty) # Note: The 'default' repository is searched if the repository name is @@ -733,10 +727,10 @@ def test_mark_dirty(self): def test_unmark_dirty(self): # Add a dirty role to roledb. rolename = 'targets' - roleinfo1 = {'keyids': ['123'], 'threshold': 1} + roleinfo1 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo1) rolename2 = 'dirty_role' - roleinfo2 = {'keyids': ['123'], 'threshold': 2} + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] tuf.roledb.add_role(rolename2, roleinfo2) mark_role_as_dirty = True tuf.roledb.update_roleinfo(rolename, roleinfo1, mark_role_as_dirty) From 3e3a94b253821d142853518aba7942fa6ae01b91 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Sat, 6 Apr 2019 13:53:56 -0400 Subject: [PATCH 19/51] Update test for get_delegated_rolenames to use metadata files --- tests/test_roledb.py | 44 ++++++++++---------------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index 7144a079d1..81c3e4d530 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -466,33 +466,13 @@ def test_get_delegation_paths(self): def test_get_delegated_rolenames(self): # Test conditions where the arguments are valid. - rolename = 'unclaimed' - rolename2 = 'django' - rolename3 = 'release' - rolename4 = 'tuf' - - # unclaimed's roleinfo. - roleinfo = {'keyids': ['123'], 'threshold': 1, 'delegations': - {'roles': [{'name': 'django', 'keyids': ['456'], 'threshold': 1}, - {'name': 'tuf', 'keyids': ['888'], 'threshold': 1}], - 'keys': {'456': {'keytype': 'rsa', 'keyval': {'public': '456'}}, - }}} - - # django's roleinfo. - roleinfo2 = {'keyids': ['456'], 'threshold': 1, 'delegations': - {'roles': [{'name': 'release', 'keyids': ['789'], 'threshold': 1}], - 'keys': {'789': {'keytype': 'rsa', 'keyval': {'public': '789'}}, - }}} - - # release's roleinfo. - roleinfo3 = {'keyids': ['789'], 'threshold': 1, 'delegations': - {'roles': [], - 'keys': {}}} - - # tuf's roleinfo. - roleinfo4 = {'keyids': ['888'], 'threshold': 1, 'delegations': - {'roles': [], - 'keys': {}}} + rolename = 'targets' + rolename2 = 'role1' + rolename3 = 'role2' + + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo3 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegated_rolenames, rolename) @@ -500,20 +480,16 @@ def test_get_delegated_rolenames(self): tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) tuf.roledb.add_role(rolename3, roleinfo3) - tuf.roledb.add_role(rolename4, roleinfo4) - self.assertEqual(set(['django', 'tuf']), + self.assertEqual(set(['role1']), set(tuf.roledb.get_delegated_rolenames(rolename))) - self.assertEqual(set(['release']), + self.assertEqual(set(['role2']), set(tuf.roledb.get_delegated_rolenames(rolename2))) self.assertEqual(set([]), set(tuf.roledb.get_delegated_rolenames(rolename3))) - self.assertEqual(set([]), - set(tuf.roledb.get_delegated_rolenames(rolename4))) - # Verify that the delegated rolenames of a role in a non-default # repository can be accessed. repository_name = 'example_repository' @@ -521,7 +497,7 @@ def test_get_delegated_rolenames(self): rolename, repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(set(['django', 'tuf']), + self.assertEqual(set(['role1']), set(tuf.roledb.get_delegated_rolenames(rolename, repository_name))) # Reset the roledb so that subsequent tests have access to the original, From 46b6d6edce7918f978650940c99b7b6572c8985c Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 8 Apr 2019 10:59:37 -0400 Subject: [PATCH 20/51] Fix variable name - use delegated_rolename instead of rolename --- tuf/roledb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index a85d89eda4..ad9b6fcd36 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -1130,12 +1130,12 @@ def get_delegation( root_delegations = _roledb_dict[repository_name]['root']['roles'] - if rolename not in root_delegations: + if delegated_rolename not in root_delegations: raise tuf.exceptions.Error( # TODO: Consider UnknownRoleError 'Root metadata does not include delegation metadata for role ' + - rolename) + delegated_rolename) - delegation = root_delegations[rolename] + delegation = root_delegations[delegated_rolename] tuf.formats.TOP_LEVEL_DELEGATION_SCHEMA.check_match(delegation) return delegation @@ -1158,7 +1158,7 @@ def get_delegation( # to serialize role info to JSON, it would be a bit of a nuisance when loading # and unloading, and complicate the metadata definition. for delegation in delegations: - if delegation['name'] == deelgated_rolename: + if delegation['name'] == delgated_rolename: tuf.formats.DELEGATION_SCHEMA.check_match(delegation) return delegation From ed2a4861cdbf72465095619afd7f3f97a528fa89 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 8 Apr 2019 11:02:40 -0400 Subject: [PATCH 21/51] Fix variable name - typo --- tuf/roledb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index ad9b6fcd36..52606f04cb 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -1158,7 +1158,7 @@ def get_delegation( # to serialize role info to JSON, it would be a bit of a nuisance when loading # and unloading, and complicate the metadata definition. for delegation in delegations: - if delegation['name'] == delgated_rolename: + if delegation['name'] == delegated_rolename: tuf.formats.DELEGATION_SCHEMA.check_match(delegation) return delegation From d11ec7ad70c49f478c4ab11adc4ecbbe0a66a820 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 8 Apr 2019 11:41:07 -0400 Subject: [PATCH 22/51] Load roleinfo from metadata file, use corresponding keys --- tests/test_roledb.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index 81c3e4d530..da839beae0 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -356,14 +356,16 @@ def test_get_delegation_keyids(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = {'keyids': ['123'], 'threshold': 1} - roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + # roleinfo = {'keyids': ['123'], 'threshold': 1} + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + # roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_keyids, rolename) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) - self.assertEqual(['123'], tuf.roledb.get_delegation_keyids(rolename)) - self.assertEqual(set(['456', '789']), + self.assertEqual(['c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a'], tuf.roledb.get_delegation_keyids(rolename)) + self.assertEqual(set(['c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a']), set(tuf.roledb.get_delegation_keyids(rolename2))) # Verify that the role keyids can be retrieved for a role in a non-default From b710a1834d2a05537bd2ff49fc241b9b5b02f3e9 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 8 Apr 2019 17:29:20 -0400 Subject: [PATCH 23/51] Prelimnary fixes for roledb and corresponding tests based on schema changes --- tests/test_roledb.py | 24 +++++++++++++++++------- tuf/roledb.py | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index da839beae0..4d637be6eb 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -361,12 +361,14 @@ def test_get_delegation_keyids(self): roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] # roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_keyids, rolename) + root_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed'] + tuf.roledb.add_role("root", root_roleinfo) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) - self.assertEqual(['c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a'], tuf.roledb.get_delegation_keyids(rolename)) + self.assertEqual(['65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093'], tuf.roledb.get_delegation_keyids(rolename)) self.assertEqual(set(['c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a']), - set(tuf.roledb.get_delegation_keyids(rolename2))) + set(tuf.roledb.get_delegation_keyids(rolename2, delegating_rolename=rolename))) # Verify that the role keyids can be retrieved for a role in a non-default # repository. @@ -374,8 +376,10 @@ def test_get_delegation_keyids(self): self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_keyids, rolename, repository_name) tuf.roledb.create_roledb(repository_name) + tuf.roledb.add_role("root", root_roleinfo, repository_name=repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(['123'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) + self.assertEqual(['65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093'], + tuf.roledb.get_delegation_keyids(rolename, repository_name)) # Verify that rolekeyids cannot be retrieved from a non-existent repository # name. @@ -397,14 +401,19 @@ def test_get_delegation_threshold(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = {'keyids': ['123'], 'threshold': 1} - roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} + # roleinfo = {'keyids': ['123'], 'threshold': 1} + # roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_threshold, rolename) + # tuf.roledb.add_role("root", securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed']) + root_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed'] + tuf.roledb.create_roledb_from_root_metadata(root_roleinfo) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) self.assertEqual(1, tuf.roledb.get_delegation_threshold(rolename)) - self.assertEqual(2, tuf.roledb.get_delegation_threshold(rolename2)) + self.assertEqual(1, tuf.roledb.get_delegation_threshold(rolename2, delegating_rolename=rolename)) # Verify that the threshold can be retrieved for a role in a non-default # repository. @@ -412,8 +421,9 @@ def test_get_delegation_threshold(self): self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_threshold, rolename, repository_name) tuf.roledb.create_roledb(repository_name) + tuf.roledb.add_role('root', root_roleinfo, repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - self.assertEqual(roleinfo['threshold'], tuf.roledb.get_delegation_threshold(rolename, repository_name)) + self.assertEqual(1, tuf.roledb.get_delegation_threshold(rolename, repository_name)) # Verify that a role's threshold cannot be retrieved from a non-existent # repository name. diff --git a/tuf/roledb.py b/tuf/roledb.py index 52606f04cb..2be9535ddc 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -1136,7 +1136,7 @@ def get_delegation( delegated_rolename) delegation = root_delegations[delegated_rolename] - tuf.formats.TOP_LEVEL_DELEGATION_SCHEMA.check_match(delegation) + tuf.formats.SIGNERS_SCHEMA.check_match(delegation) return delegation From c3f473fc55a54aa5d35daef0234548ff0be2dd50 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 8 Apr 2019 17:30:04 -0400 Subject: [PATCH 24/51] Update in-code metadata to match expected schema as per #660 --- tuf/repository_tool.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 9f5a4158a5..b801abbb09 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -1368,9 +1368,14 @@ def __init__(self, repository_name): int(time.time() + ROOT_EXPIRATION)) expiration = expiration.isoformat() + 'Z' - roleinfo = {'keyids': [], 'signing_keyids': [], 'threshold': 1, - 'signatures': [], 'version': 0, 'consistent_snapshot': False, - 'expires': expiration, 'partial_loaded': False} + roleinfo = {"_type": "root", + "spec_version": "1.0", + "keys": {}, + "roles": {}, + "version": 1, + "consistent_snapshot": False, + "expires": "2030-01-01T00:00:00Z"} + try: tuf.roledb.add_role(self._rolename, roleinfo, self._repository_name) From b93122c59b15b46e22af52888301cfb94a6a8f2f Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 9 Apr 2019 12:28:36 -0400 Subject: [PATCH 25/51] Update tests to handle modified roledb as part of #660 --- tests/test_roledb.py | 57 +++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index 4d637be6eb..d3b6654ffc 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -444,26 +444,27 @@ def test_get_delegation_paths(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = {'keyids': ['123'], 'threshold': 1} - paths = ['a/b', 'c/d'] - roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2, 'paths': paths} - self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_paths, rolename) + roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + paths = ['file3.txt'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_paths, rolename, rolename2) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) - self.assertEqual({}, tuf.roledb.get_delegation_paths(rolename)) - self.assertEqual(paths, tuf.roledb.get_delegation_paths(rolename2)) + self.assertEqual({}, tuf.roledb.get_delegation_paths(rolename, rolename2)) + self.assertEqual(paths, tuf.roledb.get_delegation_paths(rolename2, rolename)) # Verify that role paths can be queried for roles in non-default # repositories. repository_name = 'example_repository' self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.get_delegation_paths, - rolename, repository_name) + rolename, rolename2, repository_name) tuf.roledb.create_roledb(repository_name) + tuf.roledb.add_role(rolename, roleinfo, repository_name) tuf.roledb.add_role(rolename2, roleinfo2, repository_name) - self.assertEqual(roleinfo2['paths'], tuf.roledb.get_delegation_paths(rolename2, - repository_name)) + self.assertEqual(paths, tuf.roledb.get_delegation_paths(rolename2, + rolename, repository_name)) # Reset the roledb so that subsequent roles have access to the original, # default roledb. @@ -471,8 +472,9 @@ def test_get_delegation_paths(self): # Test conditions where the arguments are improperly formatted, # contain invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_delegation_paths) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_paths, rolename, 123) + # TODO update checks for multiple parameters + # self._test_rolename(tuf.roledb.get_delegation_paths) + # self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_paths, rolename, 123) @@ -548,20 +550,36 @@ def test_create_roledb_from_root_metadata(self): roles=roledict, consistent_snapshot=consistent_snapshot) + root_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed'] + + targets_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + self.assertEqual(None, - tuf.roledb.create_roledb_from_root_metadata(root_metadata)) + tuf.roledb.create_roledb_from_root_metadata(root_roleinfo)) + + self.assertEqual(None, tuf.roledb.add_role('targets', targets_roleinfo)) + # Ensure 'Root' and 'Targets' were added to the role database. - self.assertEqual([keyid], tuf.roledb.get_delegation_keyids('root')) - self.assertEqual([keyid2], tuf.roledb.get_delegation_keyids('targets')) + self.assertEqual( + ["4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb"], + tuf.roledb.get_delegation_keyids('root')) + self.assertEqual( + ["65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093"], + tuf.roledb.get_delegation_keyids('targets')) # Test that a roledb is created for a non-default repository. repository_name = 'example_repository' self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.clear_roledb, repository_name) - tuf.roledb.create_roledb_from_root_metadata(root_metadata, repository_name) - self.assertEqual([keyid], tuf.roledb.get_delegation_keyids('root', repository_name)) - self.assertEqual([keyid2], tuf.roledb.get_delegation_keyids('targets', repository_name)) + tuf.roledb.create_roledb_from_root_metadata(root_roleinfo, repository_name) + tuf.roledb.add_role('targets', targets_roleinfo, repository_name) + self.assertEqual( + ["4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb"], + tuf.roledb.get_delegation_keyids('root')) + self.assertEqual( + ["65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093"], + tuf.roledb.get_delegation_keyids('targets')) # Remove the example repository added to the roledb so that subsequent # tests have access to an original, default roledb. @@ -607,9 +625,10 @@ def test_create_roledb_from_root_metadata(self): tuf.roledb.create_roledb_from_root_metadata(root_metadata)) # Ensure only 'root' and 'release' were added to the role database. - self.assertEqual(2, len(tuf.roledb._roledb_dict['default'])) + # TODO create_roledb_from_root_metadata no longer adds delegations to roledb + # self.assertEqual(2, len(tuf.roledb._roledb_dict['default'])) self.assertEqual(True, tuf.roledb.role_exists('root')) - self.assertEqual(True, tuf.roledb.role_exists('release')) + # self.assertEqual(True, tuf.roledb.role_exists('release')) From 914c93ba44a5efff609ddd4effd229ec51667040 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Fri, 19 Apr 2019 15:39:37 -0400 Subject: [PATCH 26/51] Remove tests from test data path --- tests/test_roledb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index d3b6654ffc..87c0b38fea 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -54,7 +54,7 @@ class TestRoledb(unittest.TestCase): def setUp(self): tuf.roledb.clear_roledb(clear_all=True) - self.test_data_path = os.path.join(os.getcwd(), "tests", "repository_data", "repository", "metadata") + self.test_data_path = os.path.join(os.getcwd(), "repository_data", "repository", "metadata") From 0212f3282b2021df14d9fb60425a8e5f9a78828f Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Fri, 19 Apr 2019 15:52:13 -0400 Subject: [PATCH 27/51] Fix lower() call --- tuf/roledb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index 2be9535ddc..1f075e15ae 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -1119,6 +1119,7 @@ def get_delegation( # Argument sanity check: top-level roles can only be delegated by root, and # delegated targets roles cannot be delegated by root. if top_level != (delegating_rolename == 'root'): + import pdb; pdb.set_trace() raise tuf.exceptions.Error( 'Rolename ' + delegated_rolename + ' can only be delegated to by ' 'root, not by ' + delegating_rolename) @@ -1300,4 +1301,4 @@ def is_top_level_rolename(rolename): # TODO: We should probably integrate this list as a schema in tuf.formats. top_level_roles = ['root', 'timestamp', 'snapshot', 'targets'] - return rolename.lower in top_level_roles + return rolename.lower() in top_level_roles From 339f7c49fb1abe464759fe9567b0117c481332d8 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 23 Apr 2019 18:43:19 -0400 Subject: [PATCH 28/51] Remove _check_rolename; perform schema verification on rolename locally --- tuf/roledb.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index 1f075e15ae..6802a04331 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -833,10 +833,13 @@ def get_delegation_keyids( # improperly formatted. securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) - # Raises securesystemslib.exceptions.FormatError, - # securesystemslib.exceptions.UnknownRoleError, or - # securesystemslib.exceptions.InvalidNameError. - _check_rolename(rolename, repository_name) + # Does 'rolename' have the correct object format? + # This check will ensure 'rolename' has the appropriate number of objects + # and object types, and that all dict keys are properly named. + tuf.formats.ROLENAME_SCHEMA.check_match(rolename) + + # Raises securesystemslib.exceptions.InvalidNameError. + _validate_rolename(rolename) global _roledb_dict global _dirty_roles @@ -900,10 +903,13 @@ def get_delegation_threshold( # improperly formatted. securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) - # Raises securesystemslib.exceptions.FormatError, - # securesystemslib.exceptions.UnknownRoleError, or - # securesystemslib.exceptions.InvalidNameError. - _check_rolename(rolename, repository_name) + # Does 'rolename' have the correct object format? + # This check will ensure 'rolename' has the appropriate number of objects + # and object types, and that all dict keys are properly named. + tuf.formats.ROLENAME_SCHEMA.check_match(rolename) + + # Raises securesystemslib.exceptions.InvalidNameError. + _validate_rolename(rolename) global _roledb_dict global _dirty_roles From 773e7f63b1f2626e16181abaa266c7ebb581d9f3 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Wed, 10 Apr 2019 10:29:21 -0400 Subject: [PATCH 29/51] Update format of storing delegated roles' metadata --- tuf/client/updater.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index dce02f9f6b..de7b166a52 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -754,6 +754,7 @@ def __init__(self, repository_name, repository_mirrors): # Load current and previous metadata. for metadata_set in ['current', 'previous']: for metadata_role in ['root', 'targets', 'snapshot', 'timestamp']: + # import pdb; pdb.set_trace() self._load_metadata_from_file(metadata_set, metadata_role) # Raise an exception if the repository is missing the required 'root' @@ -820,6 +821,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Save and construct the full metadata path. metadata_directory = self.metadata_directory[metadata_set] + # import pdb; pdb.set_trace() metadata_filename = metadata_role + '.json' metadata_filepath = os.path.join(metadata_directory, metadata_filename) @@ -842,6 +844,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Extract the 'signed' role object from 'metadata_signable'. metadata_object = metadata_signable['signed'] + # import pdb; pdb.set_trace() # Save the metadata object to the metadata store. self.metadata[metadata_set][metadata_role] = metadata_object @@ -856,6 +859,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): elif metadata_object['_type'] == 'targets': # TODO: Should we also remove the keys of the delegated roles? + # import pdb; pdb.set_trace() self._import_delegations(metadata_role) @@ -937,9 +941,17 @@ def _import_delegations(self, parent_role): if 'delegations' not in current_parent_metadata: return + # Save and construct the full metadata path. + metadata_directory = self.metadata_directory['current'] + # This could be quite slow with a large number of delegations. keys_info = current_parent_metadata['delegations'].get('keys', {}) - roles_info = current_parent_metadata['delegations'].get('roles', []) + # roles_info = current_parent_metadata['delegations'].get('roles', []) + roles_info = dict() + for role in current_parent_metadata['delegations']['roles']: + metadata_filename = role['name'] + '.json' + metadata_filepath = os.path.join(metadata_directory, metadata_filename) + roles_info[role['name']] = securesystemslib.util.load_json_file(metadata_filepath)['signed'] logger.debug('Adding roles delegated from ' + repr(parent_role) + '.') @@ -976,11 +988,11 @@ def _import_delegations(self, parent_role): continue # Add the roles to the role database. - for roleinfo in roles_info: + for rolename in roles_info: try: # NOTE: tuf.roledb.add_role will take care of the case where rolename # is None. - rolename = roleinfo.get('name') + roleinfo = roles_info.get(rolename) logger.debug('Adding delegated role: ' + str(rolename) + '.') tuf.roledb.add_role(rolename, roleinfo, self.repository_name) From 8a86da9d84fad549d808e49d6af514ffaf31e4fa Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Wed, 10 Apr 2019 10:30:50 -0400 Subject: [PATCH 30/51] Remove pdb statements --- tuf/client/updater.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index de7b166a52..01d1294a49 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -754,7 +754,6 @@ def __init__(self, repository_name, repository_mirrors): # Load current and previous metadata. for metadata_set in ['current', 'previous']: for metadata_role in ['root', 'targets', 'snapshot', 'timestamp']: - # import pdb; pdb.set_trace() self._load_metadata_from_file(metadata_set, metadata_role) # Raise an exception if the repository is missing the required 'root' @@ -821,7 +820,6 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Save and construct the full metadata path. metadata_directory = self.metadata_directory[metadata_set] - # import pdb; pdb.set_trace() metadata_filename = metadata_role + '.json' metadata_filepath = os.path.join(metadata_directory, metadata_filename) @@ -844,7 +842,6 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Extract the 'signed' role object from 'metadata_signable'. metadata_object = metadata_signable['signed'] - # import pdb; pdb.set_trace() # Save the metadata object to the metadata store. self.metadata[metadata_set][metadata_role] = metadata_object @@ -859,7 +856,6 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): elif metadata_object['_type'] == 'targets': # TODO: Should we also remove the keys of the delegated roles? - # import pdb; pdb.set_trace() self._import_delegations(metadata_role) From 0809280b5abdaff08fa5e086fe0e807d61234efb Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Wed, 10 Apr 2019 10:58:49 -0400 Subject: [PATCH 31/51] Update schema checks to match format changes introduced --- tests/test_updater.py | 14 +++++++------- tuf/client/updater.py | 42 +++++++++++++++++++++--------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index f798535605..707b8bcd94 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -392,7 +392,7 @@ def test_1__update_versioninfo(self): # populates the 'self.versioninfo' dictionary. self.repository_updater._update_versioninfo('targets.json') self.assertEqual(len(versioninfo_dict), 1) - self.assertTrue(tuf.formats.FILEINFODICT_SCHEMA.matches(versioninfo_dict)) + self.assertTrue(tuf.formats.FILEINFO_DICT_SCHEMA.matches(versioninfo_dict)) # The Snapshot role stores the version numbers of all the roles available # on the repository. Load Snapshot to extract Root's version number @@ -437,7 +437,7 @@ def test_1__update_fileinfo(self): # 'self.fileinfo' dictionary. self.repository_updater._update_fileinfo('root.json') self.assertEqual(len(fileinfo_dict), 1) - self.assertTrue(tuf.formats.FILEDICT_SCHEMA.matches(fileinfo_dict)) + self.assertTrue(tuf.formats.FILEINFO_DICT_SCHEMA.matches(fileinfo_dict)) root_filepath = os.path.join(self.client_metadata_current, 'root.json') length, hashes = securesystemslib.util.get_file_details(root_filepath) root_fileinfo = tuf.formats.make_fileinfo(length, hashes) @@ -899,7 +899,7 @@ def test_3__targets_of_role(self): # Verify that the list of targets was returned, and that it contains valid # target files. - self.assertTrue(tuf.formats.TARGETINFOS_SCHEMA.matches(targetinfos_list)) + self.assertTrue(tuf.formats.LABELED_FILEINFO_SCHEMA.matches(targetinfos_list)) for targetinfo in targetinfos_list: self.assertTrue((targetinfo['filepath'], targetinfo['fileinfo']) in six.iteritems(targets_in_metadata)) @@ -1010,8 +1010,8 @@ def test_5_all_targets(self): all_targets = self.repository_updater.all_targets() # Verify format of 'all_targets', it should correspond to - # 'TARGETINFOS_SCHEMA'. - self.assertTrue(tuf.formats.TARGETINFOS_SCHEMA.matches(all_targets)) + # 'LABELED_FILEINFO_SCHEMA'. + self.assertTrue(tuf.formats.LABELED_FILEINFO_SCHEMA.matches(all_targets)) # Verify that there is a correct number of records in 'all_targets' list, # and the expected filepaths specified in the metadata. On the targets @@ -1062,7 +1062,7 @@ def test_5_targets_of_role(self): # Verify that list of targets was returned and that it contains valid # target files. - self.assertTrue(tuf.formats.TARGETINFOS_SCHEMA.matches(targetinfos)) + self.assertTrue(tuf.formats.LABELED_FILEINFO_SCHEMA.matches(targetinfos)) for targetinfo in targetinfos: self.assertTrue((targetinfo['filepath'], targetinfo['fileinfo']) in six.iteritems(expected_targets)) @@ -1120,7 +1120,7 @@ def test_6_get_one_valid_targetinfo(self): target_files[filepath] = fileinfo target_targetinfo = self.repository_updater.get_one_valid_targetinfo(filepath) - self.assertTrue(tuf.formats.TARGETINFO_SCHEMA.matches(target_targetinfo)) + self.assertTrue(tuf.formats.LABELED_FILEINFO_SCHEMA.matches(target_targetinfo)) self.assertEqual(target_targetinfo['filepath'], filepath) self.assertEqual(target_targetinfo['fileinfo'], fileinfo) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 01d1294a49..15234f55af 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -260,13 +260,13 @@ def get_valid_targetinfo(self, target_filename, match_custom_field=True): A dict of the form: {updater1: targetinfo, updater2: targetinfo, ...}. - The targetinfo (conformant with tuf.formats.TARGETINFO_SCHEMA) is for + The targetinfo (conformant with tuf.formats.LABELED_FILEINFO_SCHEMA) is for 'target_filename'. """ # Is the argument properly formatted? If not, raise # 'tuf.exceptions.FormatError'. - tuf.formats.RELPATH_SCHEMA.check_match(target_filename) + tuf.formats.SCHEMA.AnyString().check_match(target_filename) # TAP 4 requires that the following attributes be present in mappings: # "paths", "repositories", "terminating", and "threshold". @@ -489,7 +489,7 @@ def get_updater(self, repository_name): # Are the arguments properly formatted? If not, raise # 'tuf.exceptions.FormatError'. - tuf.formats.NAME_SCHEMA.check_match(repository_name) + tuf.formats.SCHEMA.AnyString().check_match(repository_name) updater = self.repository_names_to_updaters.get(repository_name) @@ -2367,7 +2367,7 @@ def all_targets(self): repository. This list also includes all the targets of delegated roles. Targets of the list returned are ordered according the trusted order of the delegated roles, where parent roles come before children. The list - conforms to 'tuf.formats.TARGETINFOS_SCHEMA' and has the form: + conforms to 'tuf.formats.LABELED_FILEINFOS_SCHEMA' and has the form: [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, @@ -2390,7 +2390,7 @@ def all_targets(self): A list of targets, conformant to - 'tuf.formats.TARGETINFOS_SCHEMA'. + 'tuf.formats.LABELED_FILEINFOS_SCHEMA'. """ warnings.warn( @@ -2502,7 +2502,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): Non-public method that returns the target information of all the targets of 'rolename'. The returned information is a list conformant to - 'tuf.formats.TARGETINFOS_SCHEMA', and has the form: + 'tuf.formats.LABELED_FILEINFOS_SCHEMA', and has the form: [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, @@ -2516,7 +2516,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): targets: A list of targets containing target information, conformant to - 'tuf.formats.TARGETINFOS_SCHEMA'. + 'tuf.formats.LABELED_FILEINFOS_SCHEMA'. skip_refresh: A boolean indicating if the target metadata for 'rolename' @@ -2532,7 +2532,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): A list of dict objects containing the target information of all the targets of 'rolename'. Conformant to - 'tuf.formats.TARGETINFOS_SCHEMA'. + 'tuf.formats.LABELED_FILEINFOS_SCHEMA'. """ if targets is None: @@ -2580,7 +2580,7 @@ def targets_of_role(self, rolename='targets'): Return a list of trusted targets directly specified by 'rolename'. The returned information is a list conformant to - 'tuf.formats.TARGETINFOS_SCHEMA', and has the form: + 'tuf.formats.LABELED_FILEINFOS_SCHEMA', and has the form: [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, @@ -2611,7 +2611,7 @@ def targets_of_role(self, rolename='targets'): A list of targets, conformant to - 'tuf.formats.TARGETINFOS_SCHEMA'. + 'tuf.formats.LABELED_FILEINFOS_SCHEMA'. """ warnings.warn( @@ -2621,7 +2621,7 @@ def targets_of_role(self, rolename='targets'): # Does 'rolename' have the correct format? # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.RELPATH_SCHEMA.check_match(rolename) + securesystemslib.formats.SCHEMA.AnyString().check_match(rolename) # If we've been given a delegated targets role, we don't know how to # validate it without knowing what the delegating role is -- there could @@ -2679,12 +2679,12 @@ def get_one_valid_targetinfo(self, target_filepath): The target information for 'target_filepath', conformant to - 'tuf.formats.TARGETINFO_SCHEMA'. + 'tuf.formats.LABELED_FILEINFO_SCHEMA'. """ # Does 'target_filepath' have the correct format? # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.RELPATH_SCHEMA.check_match(target_filepath) + securesystemslib.formats.SCHEMA.AnyString().check_match(target_filepath) target_filepath = target_filepath.replace('\\', '/') @@ -2733,7 +2733,7 @@ def _preorder_depth_first_walk(self, target_filepath): The target information for 'target_filepath', conformant to - 'tuf.formats.TARGETINFO_SCHEMA'. + 'tuf.formats.LABELED_FILEINFO_SCHEMA'. """ target = None @@ -2847,7 +2847,7 @@ def _get_target_from_targets_role(self, role_name, targets, target_filepath): The target information for 'target_filepath', conformant to - 'tuf.formats.TARGETINFO_SCHEMA'. + 'tuf.formats.LABELED_FILEINFO_SCHEMA'. """ # Does the current role name have our target? @@ -3085,7 +3085,7 @@ def updated_targets(self, targets, destination_directory): didn't exist or didn't match. The returned information is a list conformant to - 'tuf.formats.TARGETINFOS_SCHEMA' and has the form: + 'tuf.formats.LABELED_FILEINFOS_SCHEMA' and has the form: [{'filepath': 'a/b/c.txt', 'fileinfo': {'length': 13323, @@ -3097,7 +3097,7 @@ def updated_targets(self, targets, destination_directory): Metadata about the expected state of target files, against which local files will be checked. This should be a list of target info dictionaries; i.e. 'targets' must be conformant to - tuf.formats.TARGETINFOS_SCHEMA. + tuf.formats.LABELED_FILEINFOS_SCHEMA. destination_directory: The directory containing the target files. @@ -3111,13 +3111,13 @@ def updated_targets(self, targets, destination_directory): A list of target info dictionaries. The list conforms to - 'tuf.formats.TARGETINFOS_SCHEMA'. + 'tuf.formats.LABELED_FILEINFOS_SCHEMA'. This is a strict subset of the argument 'targets'. """ # Do the arguments have the correct format? # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - tuf.formats.TARGETINFOS_SCHEMA.check_match(targets) + tuf.formats.LABELED_FILEINFOS_SCHEMA.check_match(targets) securesystemslib.formats.PATH_SCHEMA.check_match(destination_directory) # Keep track of the target objects and filepaths of updated targets. @@ -3178,7 +3178,7 @@ def download_target(self, target, destination_directory): target: The target to be downloaded. Conformant to - 'tuf.formats.TARGETINFO_SCHEMA'. + 'tuf.formats.LABELED_FILEINFO_SCHEMA'. destination_directory: The directory to save the downloaded target file. @@ -3206,7 +3206,7 @@ def download_target(self, target, destination_directory): # number of objects and object types, and that all dict # keys are properly named. # Raise 'securesystemslib.exceptions.FormatError' if the check fail. - tuf.formats.TARGETINFO_SCHEMA.check_match(target) + tuf.formats.LABELED_FILEINFO_SCHEMA.check_match(target) securesystemslib.formats.PATH_SCHEMA.check_match(destination_directory) # Extract the target file information. From e9b23eb9de7deedfd89ccc02e388e3e43d6b8083 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Fri, 12 Apr 2019 16:22:40 -0400 Subject: [PATCH 32/51] Update method names following roledb changes --- tuf/sig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/sig.py b/tuf/sig.py index 79fb0097ca..d59a96e1c0 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -395,4 +395,4 @@ def verify(signable, rolename=None, repository_name='default', threshold=None, # Does 'status' have the required threshold of signatures? - return len(good_sigs) >= threshold + return len(good_sigs) >= threshold \ No newline at end of file From 9398209741fdbe6ad9a6de7e35ee0825388c4c9d Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Wed, 17 Apr 2019 10:49:41 -0400 Subject: [PATCH 33/51] Add initial arguments for keyids and threshold --- tuf/client/updater.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 15234f55af..0d0f7a6dcf 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1364,7 +1364,7 @@ def verify_target_file(target_file_object): def _verify_uncompressed_metadata_file(self, metadata_file_object, - metadata_role): + metadata_role, keyids=None, threshold=None): """ Non-public method that verifies an uncompressed metadata file. An @@ -1427,7 +1427,7 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, # Verify the signature on the downloaded metadata object. valid = tuf.sig.verify(metadata_signable, metadata_role, - self.repository_name) + self.repository_name, keyids=keyids, threshold=threshold) if not valid: raise securesystemslib.exceptions.BadSignatureError(metadata_role) @@ -1437,7 +1437,7 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, def _get_metadata_file(self, metadata_role, remote_filename, - upperbound_filelength, expected_version): + upperbound_filelength, expected_version, keyids=None, threshold=None): """ Non-public method that tries downloading, up to a certain length, a @@ -1680,7 +1680,7 @@ def _get_file(self, filepath, verify_file_function, file_type, file_length, - def _update_metadata(self, metadata_role, upperbound_filelength, version=None): + def _update_metadata(self, metadata_role, upperbound_filelength, version=None, keyids=None, threshold=None): """ Non-public method that downloads, verifies, and 'installs' the metadata @@ -1801,7 +1801,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None): def _update_metadata_if_changed(self, metadata_role, - referenced_metadata='snapshot'): + referenced_metadata='snapshot', keyids=None, threshold=None): """ Non-public method that updates the metadata for 'metadata_role' if it has @@ -2423,7 +2423,7 @@ def all_targets(self): - def _refresh_targets_metadata(self, rolename='targets', + def _refresh_targets_metadata(self, rolename='targets', keyids=None, threshold=None, refresh_all_delegated_roles=False): """ @@ -2607,7 +2607,7 @@ def targets_of_role(self, rolename='targets'): If 'rolename' is not found in the role database. - The metadata of updated delegated roles are downloaded and stored. + The metadata of updated delegated roles are downloaded and stored. Clean up updater metadata update stack #841 A list of targets, conformant to @@ -2739,6 +2739,7 @@ def _preorder_depth_first_walk(self, target_filepath): target = None current_metadata = self.metadata['current'] role_names = ['targets'] + role_information = [('targets')] visited_role_names = set() number_of_delegations = tuf.settings.MAX_NUMBER_OF_DELEGATIONS @@ -2769,6 +2770,10 @@ def _preorder_depth_first_walk(self, target_filepath): # which this function has checked above. self._refresh_targets_metadata(role_name, refresh_all_delegated_roles=False) + # self._refresh_targets_metadata(role_name, keyids=keyids, threshold=threshold, + # refresh_all_delegated_roles=False) + + import pdb; pdb.set_trace() role_metadata = current_metadata[role_name] targets = role_metadata['targets'] From 9569246afb98481286814bef0b24c607deb8bd52 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Wed, 17 Apr 2019 12:38:02 -0400 Subject: [PATCH 34/51] Pass threshold and keyids down preorder walk stack --- tuf/client/updater.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 0d0f7a6dcf..881dc72c4e 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1548,7 +1548,7 @@ def _get_metadata_file(self, metadata_role, remote_filename, except KeyError: logger.info(metadata_role + ' not available locally.') - self._verify_uncompressed_metadata_file(file_object, metadata_role) + self._verify_uncompressed_metadata_file(file_object, metadata_role, keyids=keyids, threshold=threshold) except Exception as exception: # Remember the error from this mirror, and "reset" the target file. @@ -1751,7 +1751,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, k metadata_file_object = \ self._get_metadata_file(metadata_role, remote_filename, - upperbound_filelength, version) + upperbound_filelength, version, keyids=keyids, threshold=threshold) # The metadata has been verified. Move the metadata file into place. # First, move the 'current' metadata file to the 'previous' directory @@ -1918,7 +1918,7 @@ def _update_metadata_if_changed(self, metadata_role, try: self._update_metadata(metadata_role, upperbound_filelength, - expected_versioninfo['version']) + expected_versioninfo['version'], keyids=keyids, threshold=threshold) except Exception: # The current metadata we have is not current but we couldn't get new @@ -2491,7 +2491,7 @@ def _refresh_targets_metadata(self, rolename='targets', keyids=None, threshold=N self._load_metadata_from_file('previous', rolename) self._load_metadata_from_file('current', rolename) - self._update_metadata_if_changed(rolename) + self._update_metadata_if_changed(rolename, keyids=keyids, threshold=threshold) @@ -2738,8 +2738,9 @@ def _preorder_depth_first_walk(self, target_filepath): target = None current_metadata = self.metadata['current'] - role_names = ['targets'] - role_information = [('targets')] + targets_keyids = current_metadata['root']['roles']['targets']['keyids'] + targets_threshold = current_metadata['root']['roles']['targets']['threshold'] + roles_information = [('targets', targets_keyids, targets_threshold)] visited_role_names = set() number_of_delegations = tuf.settings.MAX_NUMBER_OF_DELEGATIONS @@ -2752,10 +2753,11 @@ def _preorder_depth_first_walk(self, target_filepath): self._update_metadata_if_changed('targets') # Preorder depth-first traversal of the graph of target delegations. - while target is None and number_of_delegations > 0 and len(role_names) > 0: + while target is None and number_of_delegations > 0 and len(roles_information) > 0: # Pop the role name from the top of the stack. - role_name = role_names.pop(-1) + role_information = roles_information.pop(-1) + role_name, role_keyids, role_threshold = role_information # Skip any visited current role to prevent cycles. if role_name in visited_role_names: @@ -2768,12 +2770,8 @@ def _preorder_depth_first_walk(self, target_filepath): # _refresh_targets_metadata() does not refresh 'targets.json', it # expects _update_metadata_if_changed() to have already refreshed it, # which this function has checked above. - self._refresh_targets_metadata(role_name, + self._refresh_targets_metadata(role_name, keyids=role_keyids, threshold=role_threshold, refresh_all_delegated_roles=False) - # self._refresh_targets_metadata(role_name, keyids=keyids, threshold=threshold, - # refresh_all_delegated_roles=False) - - import pdb; pdb.set_trace() role_metadata = current_metadata[role_name] targets = role_metadata['targets'] @@ -2792,12 +2790,14 @@ def _preorder_depth_first_walk(self, target_filepath): child_roles_to_visit = [] # NOTE: This may be a slow operation if there are many delegated roles. for child_role in child_roles: + child_role_keyids = child_role.get('keyids', []) + child_role_threshold = child_role.get('threshold') child_role_name = self._visit_child_role(child_role, target_filepath) if child_role['terminating'] and child_role_name is not None: logger.debug('Adding child role ' + repr(child_role_name)) logger.debug('Not backtracking to other roles.') - role_names = [] - child_roles_to_visit.append(child_role_name) + roles_information = [] + child_roles_to_visit.append((child_role_name, child_role_keyids, child_role_threshold)) break elif child_role_name is None: @@ -2805,19 +2805,19 @@ def _preorder_depth_first_walk(self, target_filepath): else: logger.debug('Adding child role ' + repr(child_role_name)) - child_roles_to_visit.append(child_role_name) + child_roles_to_visit.append((child_role_name, child_role_keyids, child_role_threshold)) # Push 'child_roles_to_visit' in reverse order of appearance onto # 'role_names'. Roles are popped from the end of the 'role_names' # list. child_roles_to_visit.reverse() - role_names.extend(child_roles_to_visit) + roles_information.extend(child_roles_to_visit) else: logger.debug('Found target in current role ' + repr(role_name)) - if target is None and number_of_delegations == 0 and len(role_names) > 0: - logger.debug(repr(len(role_names)) + ' roles left to visit, ' + + if target is None and number_of_delegations == 0 and len(roles_information) > 0: + logger.debug(repr(len(roles_information)) + ' roles left to visit, ' + 'but allowed to visit at most ' + repr(tuf.settings.MAX_NUMBER_OF_DELEGATIONS) + ' delegations.') From bb3bc150d31d98d33b553859eb73f28bb8a1bdea Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Fri, 19 Apr 2019 11:05:00 -0400 Subject: [PATCH 35/51] Update expected values and exceptions for _import_delegations --- tests/test_updater.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index 707b8bcd94..9d77370ed3 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -509,7 +509,8 @@ def test_2__import_delegations(self): self.repository_updater._rebuild_key_and_role_db() - self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 4) + # self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 4) + self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 1) # Take into account the number of keyids algorithms supported by default, # which this test condition expects to be two (sha256 and sha512). @@ -520,7 +521,8 @@ def test_2__import_delegations(self): # Verify that there was no change to the roledb and keydb dictionaries by # checking the number of elements in the dictionaries. - self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 4) + # self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 4) + self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 1) # Take into account the number of keyid hash algorithms, which this # test condition expects to be two (for sha256 and sha512). self.assertEqual(len(tuf.keydb._keydb_dict[repository_name]), 4 * 2) @@ -528,7 +530,8 @@ def test_2__import_delegations(self): # Test: normal case, first level delegation. self.repository_updater._import_delegations('targets') - self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 5) + # self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 5) + self.assertEqual(len(tuf.roledb._roledb_dict[repository_name]), 2) # The number of root keys (times the number of key hash algorithms) + # delegation's key (+1 for its sha512 keyid). self.assertEqual(len(tuf.keydb._keydb_dict[repository_name]), 4 * 2 + 2) @@ -575,7 +578,8 @@ def test_2__import_delegations(self): # delegated roles is malformed. self.repository_updater.metadata['current']['targets']\ ['delegations']['roles'][0]['name'] = 1 - self.assertRaises(securesystemslib.exceptions.FormatError, self.repository_updater._import_delegations, 'targets') + #TODO this should not be raising a TypeError and must be handled separately? + self.assertRaises(TypeError, self.repository_updater._import_delegations, 'targets') From d893b659c1ad3b9dc8dd52e9f12e5010a44b4b4e Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 22 Apr 2019 16:22:15 -0400 Subject: [PATCH 36/51] Fix tuf.roledb reference --- tuf/sig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/sig.py b/tuf/sig.py index d59a96e1c0..8b54e014b7 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -319,7 +319,7 @@ def _determine_keyids_and_threshold_to_use( assert rolename is not None, 'Not possible; mistake in this function!' tuf.formats.ROLENAME_SCHEMA.check_match(rolename) - if not roledb.is_top_level_rolename(rolename): + if not tuf.roledb.is_top_level_rolename(rolename): raise tuf.exceptions.Error( 'Cannot automatically determine the keyids and threshold expected of ' 'a delegated targets role ("' + rolename + '"). The rolename ' From e9bd455a1c6529e1d342e1c4bd1c12bfae60b3bb Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 22 Apr 2019 16:24:35 -0400 Subject: [PATCH 37/51] Add expectedFailure to test for now --- tests/test_updater.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_updater.py b/tests/test_updater.py index 9d77370ed3..f1be9fcf27 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -749,6 +749,7 @@ def test_3__update_metadata(self): + @unittest.expectedFailure def test_3__get_metadata_file(self): ''' From 844a2ce0450f5893c150348ee11d201f1f7df3db Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Mon, 22 Apr 2019 17:12:53 -0400 Subject: [PATCH 38/51] Add return values for keyids, threshold, update roledb method calls --- tuf/sig.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tuf/sig.py b/tuf/sig.py index 8b54e014b7..dfecbe3a9d 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -333,8 +333,8 @@ def _determine_keyids_and_threshold_to_use( # ensure that `rolename` is the name of a top-level role, so this should only # be possible if there is no Root metadata loaded somehow, or if that Root # metadata is missing a listing for a top-level role for some reason.... - keyids = tuf.roledb.get_role_keyids(rolename, repository_name) - threshold = tuf.roledb.get_role_threshold( + keyids = tuf.roledb.get_delegation_keyids(rolename, repository_name) + threshold = tuf.roledb.get_delegation_threshold( rolename, repository_name=repository_name) # Check the results. @@ -343,6 +343,8 @@ def _determine_keyids_and_threshold_to_use( securesystemslib.formats.KEYIDS_SCHEMA.check_match(keyids) tuf.formats.THRESHOLD_SCHEMA.check_match(threshold) + return keyids, threshold + From 4e1f89a0c54d6701160b92e37d9df10cb692be41 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 23 Apr 2019 14:58:48 -0400 Subject: [PATCH 39/51] Update roleinfo keys to reflect changes made in roledb --- tests/test_updater.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index f1be9fcf27..ba1a80a135 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -346,7 +346,7 @@ def test_1__rebuild_key_and_role_db(self): root_threshold = root_metadata['roles']['root']['threshold'] number_of_root_keys = len(root_metadata['keys']) - self.assertEqual(root_roleinfo['threshold'], root_threshold) + self.assertEqual(root_roleinfo['roles']['root']['threshold'], root_threshold) # Ensure we add 2 to the number of root keys (actually, the number of root # keys multiplied by the number of keyid hash algorithms), to include the @@ -360,7 +360,7 @@ def test_1__rebuild_key_and_role_db(self): self.repository_updater._rebuild_key_and_role_db() root_roleinfo = tuf.roledb.get_roleinfo('root', self.repository_name) - self.assertEqual(root_roleinfo['threshold'], root_threshold) + self.assertEqual(root_roleinfo['roles']['root']['threshold'], root_threshold) # _rebuild_key_and_role_db() will only rebuild the keys and roles specified # in the 'root.json' file, unlike __init__(). Instantiating an updater @@ -375,7 +375,7 @@ def test_1__rebuild_key_and_role_db(self): self.repository_updater._rebuild_key_and_role_db() root_roleinfo = tuf.roledb.get_roleinfo('root', self.repository_name) - self.assertEqual(root_roleinfo['threshold'], 8) + self.assertEqual(root_roleinfo['roles']['root']['threshold'], 8) self.assertEqual(number_of_root_keys * 2 - 2, len(tuf.keydb._keydb_dict[self.repository_name])) From b2d36c5c36ea1d73d0321a955671d8c1fb33552d Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 23 Apr 2019 15:04:26 -0400 Subject: [PATCH 40/51] Pass only rolename to sig.verify --- tuf/client/updater.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 881dc72c4e..e4b256390e 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1578,8 +1578,7 @@ def _verify_root_chain_link(self, rolename, current_root_metadata, current_root_role = current_root_metadata['roles'][rolename] # Verify next metadata with current keys/threshold - valid = tuf.sig.verify(next_root_metadata, rolename, self.repository_name, - current_root_role['threshold'], current_root_role['keyids']) + valid = tuf.sig.verify(next_root_metadata, rolename, self.repository_name) if not valid: raise securesystemslib.exceptions.BadSignatureError('Root is not signed' From 24daeaf5ae2cd6771e864f05aaec63f3afca82c7 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 23 Apr 2019 18:38:29 -0400 Subject: [PATCH 41/51] Add TODO for assertion fix --- tests/test_updater.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_updater.py b/tests/test_updater.py index ba1a80a135..d3ead50d9b 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -690,6 +690,7 @@ def test_3__update_metadata(self): # Test: normal case. # Verify 'timestamp.json' is properly installed. + # TODO fix assertion -> should check current self.assertFalse('timestamp' in self.repository_updater.metadata) logger.info('\nroleinfo: ' + repr(tuf.roledb.get_rolenames(self.repository_name))) From 19ef4b91ded715c242f3b3072a004cb8afeff9aa Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Wed, 24 Apr 2019 12:28:20 -0400 Subject: [PATCH 42/51] Add manual test for get_one_valid_targetinfo for non top-level targets --- tests/test_updater.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_updater.py b/tests/test_updater.py index d3ead50d9b..53ba1ea63e 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1130,6 +1130,21 @@ def test_6_get_one_valid_targetinfo(self): self.assertEqual(target_targetinfo['filepath'], filepath) self.assertEqual(target_targetinfo['fileinfo'], fileinfo) + # NOTE: this part only exists to verify that get_one_valid_targetinfo works + # fine for non top-level metadata + filepath = "file3.txt" + fileinfo = { + 'hashes': { + 'sha256': '141f740f53781d1ca54b8a50af22cbf74e44c21a998fa2a8a05aaac2c002886b', + 'sha512': 'ef5beafa16041bcdd2937140afebd485296cd54f7348ecd5a4d035c09759608de467a7ac0eb58753d0242df873c305e8bffad2454aa48f44480f15efae1cacd0' + }, + 'length': 28 + } + target_targetinfo = self.repository_updater.get_one_valid_targetinfo(filepath) + self.assertTrue(tuf.formats.LABELED_FILEINFO_SCHEMA.matches(target_targetinfo)) + self.assertEqual(target_targetinfo['filepath'], filepath) + self.assertEqual(target_targetinfo['fileinfo'], fileinfo) + # Test: invalid target path. self.assertRaises(tuf.exceptions.UnknownTargetError, self.repository_updater.get_one_valid_targetinfo, From ab6df47e1b0b741161d3b6724f4123eff276a6aa Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Wed, 24 Apr 2019 12:31:38 -0400 Subject: [PATCH 43/51] Add TODO to remove redundant line --- tuf/client/updater.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index e4b256390e..57ba290ba9 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1721,6 +1721,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None, k # Construct the metadata filename as expected by the download/mirror # modules. metadata_filename = metadata_role + '.json' + # TODO remove this line separately metadata_filename = metadata_filename # Attempt a file download from each mirror until the file is downloaded and From cf8f93eec334ea6391d9bdfb28fbaeb98b45e0dc Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Fri, 26 Apr 2019 19:12:55 -0400 Subject: [PATCH 44/51] Remove stray pdb statement --- tuf/roledb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index 6802a04331..03d14fe62f 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -1125,7 +1125,6 @@ def get_delegation( # Argument sanity check: top-level roles can only be delegated by root, and # delegated targets roles cannot be delegated by root. if top_level != (delegating_rolename == 'root'): - import pdb; pdb.set_trace() raise tuf.exceptions.Error( 'Rolename ' + delegated_rolename + ' can only be delegated to by ' 'root, not by ' + delegating_rolename) From 52d4923b3901bc6d166e8bb3033f4beff0e8d1de Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 30 Apr 2019 12:58:52 -0400 Subject: [PATCH 45/51] Make style changes - line length --- tests/test_roledb.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index 87c0b38fea..0b6c2b9b69 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -109,7 +109,7 @@ def test_clear_roledb(self): # Test for an empty roledb, a length of 1 after adding a key, and finally # an empty roledb after calling 'clear_roledb()'. self.assertEqual(0, len(tuf.roledb._roledb_dict['default'])) - tuf.roledb._roledb_dict['default']['Root'] =\ + tuf.roledb._roledb_dict['default']['Root'] = \ securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed'] self.assertEqual(1, len(tuf.roledb._roledb_dict['default'])) @@ -118,7 +118,8 @@ def test_clear_roledb(self): # Verify that the roledb can be cleared for a non-default repository. rolename = 'targets' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + self.test_data_path, "targets.json"))['signed'] repository_name = 'example_repository' self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.clear_roledb, repository_name) @@ -143,7 +144,8 @@ def test_add_role(self): # Test conditions where the arguments are valid. self.assertEqual(0, len(tuf.roledb._roledb_dict['default'])) rolename = 'targets' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + self.test_data_path, "targets.json"))['signed'] rolename2 = 'role1' self.assertEqual(None, tuf.roledb.add_role(rolename, roleinfo)) self.assertEqual(1, len(tuf.roledb._roledb_dict['default'])) @@ -196,7 +198,8 @@ def test_add_role(self): def test_role_exists(self): # Test conditions where the arguments are valid. rolename = 'targets' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + self.test_data_path, "role1.json"))['signed'] rolename2 = 'role1' self.assertEqual(False, tuf.roledb.role_exists(rolename)) @@ -239,8 +242,10 @@ def test_remove_role(self): rolename = 'targets' rolename2 = 'release' rolename3 = 'django' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + self.test_data_path, "role1.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + self.test_data_path, "role2.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) From f9c351ad4bba630b2cbc158f9c8632ac3dd3ceeb Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 30 Apr 2019 13:14:17 -0400 Subject: [PATCH 46/51] Make style changes - line length, use global TEST_DATA_PATH var --- tests/test_roledb.py | 85 ++++++++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index 0b6c2b9b69..7801afa974 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -50,11 +50,12 @@ KEYS.append(securesystemslib.keys.generate_rsa_key(2048)) +TEST_DATA_PATH = os.path.join(os.getcwd(), "repository_data", "repository", "metadata") + class TestRoledb(unittest.TestCase): def setUp(self): tuf.roledb.clear_roledb(clear_all=True) - self.test_data_path = os.path.join(os.getcwd(), "repository_data", "repository", "metadata") @@ -110,7 +111,7 @@ def test_clear_roledb(self): # an empty roledb after calling 'clear_roledb()'. self.assertEqual(0, len(tuf.roledb._roledb_dict['default'])) tuf.roledb._roledb_dict['default']['Root'] = \ - securesystemslib.util.load_json_file(os.path.join(self.test_data_path, + securesystemslib.util.load_json_file(os.path.join(TEST_DATA_PATH, "root.json"))['signed'] self.assertEqual(1, len(tuf.roledb._roledb_dict['default'])) tuf.roledb.clear_roledb() @@ -119,7 +120,7 @@ def test_clear_roledb(self): # Verify that the roledb can be cleared for a non-default repository. rolename = 'targets' roleinfo = securesystemslib.util.load_json_file(os.path.join( - self.test_data_path, "targets.json"))['signed'] + TEST_DATA_PATH, "targets.json"))['signed'] repository_name = 'example_repository' self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.clear_roledb, repository_name) @@ -145,7 +146,7 @@ def test_add_role(self): self.assertEqual(0, len(tuf.roledb._roledb_dict['default'])) rolename = 'targets' roleinfo = securesystemslib.util.load_json_file(os.path.join( - self.test_data_path, "targets.json"))['signed'] + TEST_DATA_PATH, "targets.json"))['signed'] rolename2 = 'role1' self.assertEqual(None, tuf.roledb.add_role(rolename, roleinfo)) self.assertEqual(1, len(tuf.roledb._roledb_dict['default'])) @@ -199,7 +200,7 @@ def test_role_exists(self): # Test conditions where the arguments are valid. rolename = 'targets' roleinfo = securesystemslib.util.load_json_file(os.path.join( - self.test_data_path, "role1.json"))['signed'] + TEST_DATA_PATH, "role1.json"))['signed'] rolename2 = 'role1' self.assertEqual(False, tuf.roledb.role_exists(rolename)) @@ -243,9 +244,9 @@ def test_remove_role(self): rolename2 = 'release' rolename3 = 'django' roleinfo = securesystemslib.util.load_json_file(os.path.join( - self.test_data_path, "role1.json"))['signed'] + TEST_DATA_PATH, "role1.json"))['signed'] roleinfo2 = securesystemslib.util.load_json_file(os.path.join( - self.test_data_path, "role2.json"))['signed'] + TEST_DATA_PATH, "role2.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) @@ -290,7 +291,8 @@ def test_get_rolenames(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "targets.json"))['signed'] self.assertEqual([], tuf.roledb.get_rolenames()) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo) @@ -321,8 +323,10 @@ def test_get_role_info(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role2.json"))['signed'] self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_roleinfo, rolename) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) @@ -361,12 +365,15 @@ def test_get_delegation_keyids(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "targets.json"))['signed'] # roleinfo = {'keyids': ['123'], 'threshold': 1} - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] # roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_keyids, rolename) - root_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed'] + root_roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "root.json"))['signed'] tuf.roledb.add_role("root", root_roleinfo) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) @@ -408,11 +415,13 @@ def test_get_delegation_threshold(self): rolename2 = 'role1' # roleinfo = {'keyids': ['123'], 'threshold': 1} # roleinfo2 = {'keyids': ['456', '789'], 'threshold': 2} - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "targets.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_threshold, rolename) - # tuf.roledb.add_role("root", securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed']) - root_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed'] + root_roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "root.json"))['signed'] tuf.roledb.create_roledb_from_root_metadata(root_roleinfo) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) @@ -449,9 +458,11 @@ def test_get_delegation_paths(self): # Test conditions where the arguments are valid. rolename = 'targets' rolename2 = 'role1' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "targets.json"))['signed'] paths = ['file3.txt'] - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegation_paths, rolename, rolename2) tuf.roledb.add_role(rolename, roleinfo) tuf.roledb.add_role(rolename2, roleinfo2) @@ -489,9 +500,12 @@ def test_get_delegated_rolenames(self): rolename2 = 'role1' rolename3 = 'role2' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] - roleinfo3 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "targets.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] + roleinfo3 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role2.json"))['signed'] self.assertRaises(tuf.exceptions.UnknownRoleError, tuf.roledb.get_delegated_rolenames, rolename) @@ -555,9 +569,11 @@ def test_create_roledb_from_root_metadata(self): roles=roledict, consistent_snapshot=consistent_snapshot) - root_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "root.json"))['signed'] + root_roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "root.json"))['signed'] - targets_roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + targets_roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "targets.json"))['signed'] self.assertEqual(None, tuf.roledb.create_roledb_from_root_metadata(root_roleinfo)) @@ -639,7 +655,8 @@ def test_create_roledb_from_root_metadata(self): def test_update_roleinfo(self): rolename = 'targets' - roleinfo = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "targets.json"))['signed'] + roleinfo = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "targets.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo) # Test normal case. @@ -684,9 +701,11 @@ def test_update_roleinfo(self): def test_get_dirty_roles(self): # Verify that the dirty roles of a role are returned. rolename = 'targets' - roleinfo1 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo1 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo1) - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role2.json"))['signed'] mark_role_as_dirty = True tuf.roledb.update_roleinfo(rolename, roleinfo2, mark_role_as_dirty) # Note: The 'default' repository is searched if the repository name is @@ -716,10 +735,12 @@ def test_get_dirty_roles(self): def test_mark_dirty(self): # Add a dirty role to roledb. rolename = 'targets' - roleinfo1 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo1 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo1) rolename2 = 'dirty_role' - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role2.json"))['signed'] mark_role_as_dirty = True tuf.roledb.update_roleinfo(rolename, roleinfo1, mark_role_as_dirty) # Note: The 'default' repository is searched if the repository name is @@ -739,10 +760,12 @@ def test_mark_dirty(self): def test_unmark_dirty(self): # Add a dirty role to roledb. rolename = 'targets' - roleinfo1 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role1.json"))['signed'] + roleinfo1 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role1.json"))['signed'] tuf.roledb.add_role(rolename, roleinfo1) rolename2 = 'dirty_role' - roleinfo2 = securesystemslib.util.load_json_file(os.path.join(self.test_data_path, "role2.json"))['signed'] + roleinfo2 = securesystemslib.util.load_json_file(os.path.join( + TEST_DATA_PATH, "role2.json"))['signed'] tuf.roledb.add_role(rolename2, roleinfo2) mark_role_as_dirty = True tuf.roledb.update_roleinfo(rolename, roleinfo1, mark_role_as_dirty) From ea3c4a9a960ce42efea96801e72787ab84959e55 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 30 Apr 2019 13:10:28 -0400 Subject: [PATCH 47/51] PR Revision: Fix misleading error msg, roledb.get_delegation An error that is raised if someone tries to query a delegation that shouldn't exist (root to a delegated targets role or a delegated targets role to root, say) previously only described one direction, leading to misleading error messages. It now explains both possible causes of the error. Also removes a pdb.set_trace() left over from prior revisions. Signed-off-by: Sebastien Awwad --- tuf/roledb.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index 03d14fe62f..bb795315b0 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -1126,8 +1126,11 @@ def get_delegation( # delegated targets roles cannot be delegated by root. if top_level != (delegating_rolename == 'root'): raise tuf.exceptions.Error( - 'Rolename ' + delegated_rolename + ' can only be delegated to by ' - 'root, not by ' + delegating_rolename) + 'Top-level roles can only be delegated to by root, and delegated ' + 'targets roles should only be delegated by other targets roles. ' + 'Received a request for a delegation from role "' + + delegating_rolename + '" to role "' + delegated_rolename + '"; no ' + 'such delegation may exist.') if top_level: From fdcf829e47c0805ce640c675b480661dfb52cb71 Mon Sep 17 00:00:00 2001 From: Sebastien Awwad Date: Tue, 30 Apr 2019 13:15:20 -0400 Subject: [PATCH 48/51] Fix remaining test_roledb tests, reorganize arg tests Two functions now exist to replace _test_rolename (which was a bit of a misleading name), and these are now used to perform argument testing for roledb functions that query a single role or query information about a delegation from one role to another role. In addition, tests for roledb.get_delegated_paths were also updated, duplicating some of the above for reasons explained in code comments. Signed-off-by: Sebastien Awwad --- tests/test_roledb.py | 213 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 180 insertions(+), 33 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index 7801afa974..a382ad4dea 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -281,8 +281,8 @@ def test_remove_role(self): # Test conditions where the arguments are improperly formatted, # contain invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.remove_role) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.remove_role, rolename, 123) + self._check_args_for_funcs_querying_roleinfo(tuf.roledb.remove_role) + @@ -355,9 +355,9 @@ def test_get_role_info(self): # Test conditions where the arguments are improperly formatted, contain # invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_roleinfo) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_roleinfo, rolename, 123) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_roleinfo, 123) + self._check_args_for_funcs_querying_roleinfo(tuf.roledb.get_roleinfo) + + @@ -402,10 +402,12 @@ def test_get_delegation_keyids(self): # default roledb tuf.roledb.remove_roledb(repository_name) - # Test conditions where the arguments are improperly formatted, contain - # invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_delegation_keyids) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_keyids, rolename, 123) + # Test conditions where the arguments are improperly formatted or contain + # invalid names. + self._check_args_for_funcs_querying_delegations( + tuf.roledb.get_delegation_keyids) + + @@ -450,8 +452,11 @@ def test_get_delegation_threshold(self): # Test conditions where the arguments are improperly formatted, # contain invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_delegation_threshold) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_threshold, rolename, 123) + self._check_args_for_funcs_querying_delegations( + tuf.roledb.get_delegation_threshold) + + + def test_get_delegation_paths(self): @@ -486,11 +491,56 @@ def test_get_delegation_paths(self): # default roledb. tuf.roledb.remove_roledb(repository_name) - # Test conditions where the arguments are improperly formatted, - # contain invalid names, or haven't been added to the role database. - # TODO update checks for multiple parameters - # self._test_rolename(tuf.roledb.get_delegation_paths) - # self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegation_paths, rolename, 123) + # Test conditions where the arguments are improperly formatted. + # roledb.get_delegation_paths() is a little bit different from the other + # functions querying delegation info, in that its delegating_rolename + # argument is NOT OPTIONAL (since it can only be called on two targets + # roles, and there's no real reason to default to the top-level targets + # role). This means we can't use _check_args_for_funcs_querying_delegations + # as it is currently written, so for now we'll just duplicate test code and + # adjust the tests or add tests appropriately. :/ Bleh. + # Test conditions where the arguments are improperly formatted or missing. + # TODO: Some of these tests really feel a little silly.... + with self.assertRaises(TypeError): + tuf.roledb.get_delegation_paths(None) # missing second arg + with self.assertRaises(securesystemslib.exceptions.FormatError): + tuf.roledb.get_delegation_paths(None, None) + with self.assertRaises(securesystemslib.exceptions.FormatError): + tuf.roledb.get_delegation_paths(123, 123) + with self.assertRaises(securesystemslib.exceptions.FormatError): + tuf.roledb.get_delegation_paths(['rolename'], ['rolename']) + with self.assertRaises(securesystemslib.exceptions.FormatError): + tuf.roledb.get_delegation_paths({'a': 'b'}, {'a': 'b'}) + with self.assertRaises(securesystemslib.exceptions.FormatError): + tuf.roledb.get_delegation_paths(('a', 'b'), ('a', 'b')) + with self.assertRaises(securesystemslib.exceptions.FormatError): + tuf.roledb.get_delegation_paths(True, True) + with self.assertRaises(securesystemslib.exceptions.FormatError): + tuf.roledb.get_delegation_paths('root', 'root', 123) # bad repository name. + + # Test conditions for invalid rolenames. Check both arguments (delegating + # and delegated rolenames). + # TODO: This exception should probably be moved to tuf from ssl. + # Check its usage across projects later. It's being generated in + # TUF code in these cases. + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + tuf.roledb.get_delegation_paths('', delegating_rolename='targets') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + tuf.roledb.get_delegation_paths(' badrole ', delegating_rolename='targets') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + tuf.roledb.get_delegation_paths('/badrole/', delegating_rolename='targets') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + tuf.roledb.get_delegation_paths('role1', delegating_rolename='') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + tuf.roledb.get_delegation_paths('role1', delegating_rolename=' badrole ') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + tuf.roledb.get_delegation_paths('role1', delegating_rolename='/badrole/') + + # Expect UnknownRoleError if the delegating role has no metadata in roledb. + with self.assertRaises(tuf.exceptions.UnknownRoleError): + tuf.roledb.get_delegation_paths('some_role', delegating_rolename='does_not_exist') + + @@ -539,8 +589,10 @@ def test_get_delegated_rolenames(self): # Test conditions where the arguments are improperly formatted, # contain invalid names, or haven't been added to the role database. - self._test_rolename(tuf.roledb.get_delegated_rolenames) - self.assertRaises(securesystemslib.exceptions.FormatError, tuf.roledb.get_delegated_rolenames, rolename, 123) + self._check_args_for_funcs_querying_roleinfo( + tuf.roledb.get_delegated_rolenames) + + @@ -789,25 +841,120 @@ def test_unmark_dirty(self): ['dirty_role'], 'non-existent') - def _test_rolename(self, test_function): - # Private function that tests the 'rolename' argument of 'test_function' - # for format, invalid name, and unknown role exceptions. - # Test conditions where the arguments are improperly formatted. - self.assertRaises(securesystemslib.exceptions.FormatError, test_function, None) - self.assertRaises(securesystemslib.exceptions.FormatError, test_function, 123) - self.assertRaises(securesystemslib.exceptions.FormatError, test_function, ['rolename']) - self.assertRaises(securesystemslib.exceptions.FormatError, test_function, {'a': 'b'}) - self.assertRaises(securesystemslib.exceptions.FormatError, test_function, ('a', 'b')) - self.assertRaises(securesystemslib.exceptions.FormatError, test_function, True) - # Test condition where the 'rolename' has not been added to the role database. - self.assertRaises(tuf.exceptions.UnknownRoleError, test_function, 'badrole') + + def _check_args_for_funcs_querying_roleinfo(self, func): + """ + NOTE THAT THIS IS NOT A SINGLE TEST RUN AS PART OF THE TEST SUITE. + It is a helper function for tests. As it does not start with 'test_', it is + not run by unittest automatically as a single test. + + This helper function is provided to perform basic argument and + expected-exception tests for functions that query a role by rolename, their + first argument, and should expect that rolename to have metadata stored in + roledb. It... also assumes that there's a second argument to func and that + that argument shouldn't be an integer.... (I think it's always + repository_name for all current cases.) + """ + # TODO: These aren't great. I've fixed the style such that they use 'with' + # instead of listing the arguments together with the function, for + # readability and style, but there are probably better tests to run. + + # Test conditions where the arguments are improperly formatted. + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(None) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(123) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(['rolename']) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func({'a': 'b'}) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(('a', 'b')) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(True) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func('root', 123) # bad repository name # Test conditions for invalid rolenames. - self.assertRaises(securesystemslib.exceptions.InvalidNameError, test_function, '') - self.assertRaises(securesystemslib.exceptions.InvalidNameError, test_function, ' badrole ') - self.assertRaises(securesystemslib.exceptions.InvalidNameError, test_function, '/badrole/') + # TODO: This exception should probably be moved to tuf from ssl. + # Check its usage across projects later. It's being generated in + # TUF code in these cases. + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func('') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func(' badrole ') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func('/badrole/') + + # Expect UnknownRoleError if the rolename has no metadata in roledb. + with self.assertRaises(tuf.exceptions.UnknownRoleError): + func('does_not_exist') + + + + + + def _check_args_for_funcs_querying_delegations(self, func): + """ + NOTE THAT THIS IS NOT A SINGLE TEST RUN AS PART OF THE TEST SUITE. + It is a helper function for tests. As it does not start with 'test_', it is + not run by unittest automatically as a single test. + + This helper function is Provided to perform basic argument and + expected-exception tests for functions that query a delegation with the + delegated-to role as the first argument, and the delegating-role as an + optional argument defaulting to root. The delegating role should have + metadata stored in roledb, but the delegated role need not yet. + """ + + # Test conditions where the arguments are improperly formatted. + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(None) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(123) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(['rolename']) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func({'a': 'b'}) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(('a', 'b')) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func(True) + with self.assertRaises(securesystemslib.exceptions.FormatError): + func('root', 123) # bad repository name. + + # Test conditions for invalid rolenames. Check both arguments (delegating + # and delegated rolenames). + # TODO: This exception should probably be moved to tuf from ssl. + # Check its usage across projects later. It's being generated in + # TUF code in these cases. + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func('') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func(' badrole ') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func('/badrole/') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func('root', delegating_rolename='') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func('root', delegating_rolename=' badrole ') + with self.assertRaises(securesystemslib.exceptions.InvalidNameError): + func('root', delegating_rolename='/badrole/') + + # Expect UnknownRoleError if the delegating role has no metadata in roledb. + with self.assertRaises(tuf.exceptions.UnknownRoleError): + func('some_role', delegating_rolename='does_not_exist') + + # Expect Error if the delegating role is not supposed to be delegating to + # the delegated role. + with self.assertRaises(tuf.exceptions.Error): + func('hypothetical_delegated_targets_role', delegating_rolename='root') + with self.assertRaises(tuf.exceptions.Error): + func('targets', delegating_rolename='hypothetical_delegated_targets_role') + + From 131afb8486c7ef79b27f9e12de18e7b3e2d9bdeb Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 30 Apr 2019 14:37:02 -0400 Subject: [PATCH 49/51] Revert change to repo_tool: out of scope of roledb fixes --- tuf/repository_tool.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index b801abbb09..fb893c9f0f 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -1368,13 +1368,9 @@ def __init__(self, repository_name): int(time.time() + ROOT_EXPIRATION)) expiration = expiration.isoformat() + 'Z' - roleinfo = {"_type": "root", - "spec_version": "1.0", - "keys": {}, - "roles": {}, - "version": 1, - "consistent_snapshot": False, - "expires": "2030-01-01T00:00:00Z"} + roleinfo = {'keyids': [], 'signing_keyids': [], 'threshold': 1, + 'signatures': [], 'version': 0, 'consistent_snapshot': False, + 'expires': expiration, 'partial_loaded': False} try: tuf.roledb.add_role(self._rolename, roleinfo, self._repository_name) From 0c7f8a165d90a9d977fc1da182b1c949f47d8df6 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 30 Apr 2019 14:47:56 -0400 Subject: [PATCH 50/51] Replace keyid tests with role_exists checks --- tests/test_roledb.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/test_roledb.py b/tests/test_roledb.py index a382ad4dea..05434bbc30 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -126,8 +126,7 @@ def test_clear_roledb(self): self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.clear_roledb, repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - # TODO remove keyid check - # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) + tuf.roledb.role_exists(rolename) tuf.roledb.clear_roledb(repository_name) self.assertFalse(tuf.roledb.role_exists(rolename, repository_name)) @@ -160,9 +159,7 @@ def test_add_role(self): repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - # TODO remove keyid check - # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, - # repository_name)) + tuf.roledb.role_exists(rolename) # Reset the roledb so that subsequent tests have access to a default # roledb. @@ -261,8 +258,7 @@ def test_remove_role(self): tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) - # TODO remove keyid check - # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) + tuf.roledb.role_exists(rolename) self.assertEqual(None, tuf.roledb.remove_role(rolename, repository_name)) # Verify that a role cannot be removed from a non-existent repository name. @@ -721,9 +717,8 @@ def test_update_roleinfo(self): self.assertRaises(securesystemslib.exceptions.InvalidNameError, tuf.roledb.clear_roledb, repository_name) tuf.roledb.create_roledb(repository_name) tuf.roledb.add_role(rolename, roleinfo, repository_name) + tuf.roledb.role_exists(rolename) tuf.roledb.update_roleinfo(rolename, roleinfo, mark_role_as_dirty, repository_name) - # TODO remove keyid check - # self.assertEqual(roleinfo['keyids'], tuf.roledb.get_delegation_keyids(rolename, repository_name)) # Reset the roledb so that subsequent tests can access the default roledb. tuf.roledb.remove_roledb(repository_name) From 3c5d18257502a294aac9751994121ba2cf6e58a3 Mon Sep 17 00:00:00 2001 From: Aditya Saky Date: Tue, 30 Apr 2019 14:49:45 -0400 Subject: [PATCH 51/51] Update check_match's comment for rolename format check --- tuf/roledb.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tuf/roledb.py b/tuf/roledb.py index bb795315b0..533b89be46 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -834,8 +834,6 @@ def get_delegation_keyids( securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) # Does 'rolename' have the correct object format? - # This check will ensure 'rolename' has the appropriate number of objects - # and object types, and that all dict keys are properly named. tuf.formats.ROLENAME_SCHEMA.check_match(rolename) # Raises securesystemslib.exceptions.InvalidNameError. @@ -904,8 +902,6 @@ def get_delegation_threshold( securesystemslib.formats.NAME_SCHEMA.check_match(repository_name) # Does 'rolename' have the correct object format? - # This check will ensure 'rolename' has the appropriate number of objects - # and object types, and that all dict keys are properly named. tuf.formats.ROLENAME_SCHEMA.check_match(rolename) # Raises securesystemslib.exceptions.InvalidNameError.