Fix #322 freeze attack by detecting expiry of stale metadata files

tests/test_indefinite_freeze_attack.py

@@ -15,6 +15,15 @@
     than verifying text output), use pre-generated repository files, and
     discontinue use of the old repository tools. -vladimir.v.diaz
 
+  March 9, 2016.
+    Additional test added relating to issue:
+    https://github.com/theupdateframework/tuf/issues/322
+    If a metadata file is not updated (no indication of a new version
+    available), the expiration of the pre-existing, locally trusted metadata
+    must still be detected. This additional test complains if such does not
+    occur, and accompanies code in tuf.client.updater:refresh() to detect it.
+    -sebastien.awwad
+
 <Copyright>
   See LICENSE for licensing information.
 
@@ -171,12 +180,26 @@ def tearDown(self):
 
 
   def test_without_tuf(self):
-    # Scenario:
+    # Two tests are conducted here.
+    # Test 1: If an expired timestamp is downloaded, is it recognized as such?
+    # Test 2: If we find that the timestamp acquired from a mirror indicates
+    #         that there is no new snapshot file, and our current snapshot
+    #         file is expired, is it recognized as such?
+
+    # Test 2 Begin:
+    #
+    # See description of scenario in Test 2 in the test_with_tuf method.
+    # Without TUF, Test 2 is tantamount to Test 1, and so we skip this test.
+
+
+    # Test 1 Begin:
+    #
     # 'timestamp.json' specifies the latest version of the repository files.
     # A client should only accept the same version of this file up to a certain
     # point, or else it cannot detect that new files are available for download.
     # Modify the repository's timestamp.json' so that it expires soon, copy it
     # over the to client, and attempt to re-fetch the same expired version. 
+    #
     # A non-TUF client (without a way to detect when metadata has expired) is
     # expected to download the same version, and thus the same outdated files.
     # Verify that the same file size and hash of 'timestamp.json' is downloaded.
@@ -216,11 +239,114 @@ def test_without_tuf(self):
     self.assertEqual(download_fileinfo, fileinfo)
 
 
-
   def test_with_tuf(self):
-    # The same scenario outlined in test_without_tuf() is followed here, except
-    # with a TUF client.  The TUF client performs a refresh of top-level
-    # metadata, which also includes 'timestamp.json'.
+    # Two tests are conducted here.
+    # Test 1: If an expired timestamp is downloaded, is it recognized as such?
+    # Test 2: If we find that the timestamp acquired from a mirror indicates
+    #         that there is no new snapshot file, and our current snapshot
+    #         file is expired, is it recognized as such?
+
+
+    # Test 2 Begin:
+    #
+    # Addresses this issue: https://github.com/theupdateframework/tuf/issues/322
+    #
+    # If time has passed and our snapshot (or any targets role) is expired, and
+    # the mirror whose timestamp we fetched doesn't indicate the existence of a
+    # new snapshot version, we still need to check that it's expired and notify
+    # the software update system / application / user. This test creates that
+    # scenario. The correct behavior is to raise an exception.
+    #
+    # Background: Expiration checks (updater._ensure_not_expired) were
+    # previously conducted when the metadata file was downloaded. If no new
+    # metadata file was downloaded, no expiry check would occur. In particular, 
+    # while root was checked for expiration at the beginning of each
+    # updater.refresh() cycle, and timestamp was always checked because it was
+    # always fetched, snapshot and targets were never checked if the user did 
+    # not receive evidence that they had changed.
+    # That bug was fixed and this test tests that fix going forward.
+
+    timestamp_path = os.path.join(self.repository_directory, 'metadata',
+                                  'timestamp.json')
+
+    # Modify the timestamp file on the remote repository.  'timestamp.json'
+    # must be properly updated and signed with 'repository_tool.py', otherwise
+    # the client will reject it as invalid metadata.  The resulting
+    # 'timestamp.json' should be valid metadata, but expired (as intended).
+    repository = repo_tool.load_repository(self.repository_directory)
+
+    key_file = os.path.join(self.keystore_directory, 'timestamp_key') 
+    timestamp_private = repo_tool.import_rsa_privatekey_from_file(key_file,
+                                                                  'password')
+
+    repository.timestamp.load_signing_key(timestamp_private)
+
+    # Load snapshot keys.
+    key_file = os.path.join(self.keystore_directory, 'snapshot_key') 
+    snapshot_private = repo_tool.import_rsa_privatekey_from_file(key_file,
+                                                                  'password')
+    repository.snapshot.load_signing_key(snapshot_private)
+
+    # Load root keys.
+    key_file = os.path.join(self.keystore_directory, 'root_key')
+    root_private = repo_tool.import_rsa_privatekey_from_file(key_file,
+                                                                  'password')
+    repository.root.load_signing_key(root_private)
+
+    time.sleep(0.1)
+
+    # Expire snapshot in 7s
+    datetime_object = tuf.formats.unix_timestamp_to_datetime(int(time.time() +
+                                                                 7))
+    repository.snapshot.expiration = datetime_object
+
+    # Now write to the repository
+    repository.write()
+    # And move the staged metadata to the "live" metadata.
+    time.sleep(0.1)
+    shutil.rmtree(os.path.join(self.repository_directory, 'metadata'))
+    shutil.copytree(os.path.join(self.repository_directory, 'metadata.staged'),
+                    os.path.join(self.repository_directory, 'metadata'))
+
+    # Refresh metadata on the client. For this refresh, all data is not expired.
+    time.sleep(0.1)
+    logger.info('Test: Refreshing #1 - Initial metadata refresh occurring.')
+    self.repository_updater.refresh()
+    logger.info("Test: Refreshed #1 - Initial metadata refresh occurred. Now "
+                "sleeping 6s.")
+
+    # Sleep for at least 7 seconds to ensure 'repository.snapshot.expiration'
+    # is reached.
+    time.sleep(7)
+    logger.info("Test: Refreshing #2 - Now trying to refresh again after "
+      " snapshot expiry.")
+    try:
+      self.repository_updater.refresh() # We expect this to fail!
+
+    except tuf.ExpiredMetadataError as e:
+      logger.info("Test: Refresh #2 - failed as expected. Stale-expired "
+                  "snapshot case generated a tuf.ExpiredMetadataError exception"
+                  ". Test pass.")
+    #I think that I only expect tuf.ExpiredMetadata error here. A
+    # NoWorkingMirrorError indicates something else in this case - unavailable
+    # repo, for example.
+    else:
+      self.fail("TUF failed to detect expired stale snapshot metadata. Freeze.")
+
+
+
+
+    # Test 1 Begin:
+    #
+    # 'timestamp.json' specifies the latest version of the repository files.
+    # A client should only accept the same version of this file up to a certain
+    # point, or else it cannot detect that new files are available for download.
+    # Modify the repository's timestamp.json' so that it expires soon, copy it
+    # over the to client, and attempt to re-fetch the same expired version.
+
+    # The same scenario as in test_without_tuf() is followed here, except with a
+    # TUF client. The TUF client performs a refresh of top-level metadata, which
+    # includes 'timestamp.json'.
 
     timestamp_path = os.path.join(self.repository_directory, 'metadata',
                                   'timestamp.json')
@@ -237,8 +363,8 @@ def test_with_tuf(self):
 
     repository.timestamp.load_signing_key(timestamp_private)
 
-    # expire in 1 second.
-    datetime_object = tuf.formats.unix_timestamp_to_datetime(int(time.time() + 1))
+    # expire in 4 seconds.
+    datetime_object = tuf.formats.unix_timestamp_to_datetime(int(time.time() + 4))
     repository.timestamp.expiration = datetime_object
     repository.write()
 
@@ -248,15 +374,19 @@ def test_with_tuf(self):
                     os.path.join(self.repository_directory, 'metadata'))
 
     # Verify that the TUF client detects outdated metadata and refuses to
-    # continue the update process.  Sleep for at least 2 seconds to ensure
+    # continue the update process.  Sleep for at least 4 seconds to ensure
     # 'repository.timestamp.expiration' is reached.
-    time.sleep(2)
+    time.sleep(4)
     try:
       self.repository_updater.refresh()
 
     except tuf.NoWorkingMirrorError as e:
       for mirror_url, mirror_error in six.iteritems(e.mirror_errors):
         self.assertTrue(isinstance(mirror_error, tuf.ExpiredMetadataError))
+
+    else:
+      self.fail("TUF failed to detect expired stale timestamp metadata. Freeze "
+        "attack successful.")
 
 
 if __name__ == '__main__':

tuf/client/updater.py

@@ -563,15 +563,23 @@ def _import_delegations(self, parent_role):
   def refresh(self, unsafely_update_root_if_necessary=True):
     """
     <Purpose>
-      Update the latest copies of the metadata for the top-level roles.
-      The update request process follows a specific order to ensure the
-      metadata files are securely updated.
-
-      The client would call refresh() prior to requesting target file
-      information.  Calling refresh() ensures target methods, like
-      all_targets() and target(), refer to the latest available content.
-      The latest copies, according to the currently trusted top-level metadata,
-      of delegated metadata are downloaded and updated by the target methods.
+      Update the latest copies of the metadata for the top-level roles. The
+      update request process follows a specific order to ensure the metadata
+      files are securely updated: timestamp -> snapshot -> root -> targets.
+
+      Delegated metadata is not refreshed by this method. After this method is
+      called, the use of target methods (e.g., all_targets(), targets_of_role(),
+      or target()) will update delegated metadata.
+      Calling refresh() ensures that top-level metadata is up-to-date, so that
+      the target methods can refer to the latest available content. Thus,
+      refresh() should always be called by the client before any requests of
+      target file information.
+
+      The expiration time for downloaded metadata is also verified.
+
+      If the refresh fails for any reason, it will be retried once after first
+      attempting to update the root metadata file. Only then will the exceptions
+      listed here potentially be raised.
 
     <Arguments>
       unsafely_update_root_if_necessary:
@@ -584,7 +592,9 @@ def refresh(self, unsafely_update_root_if_necessary=True):
         If the metadata for any of the top-level roles cannot be updated.
 
       tuf.ExpiredMetadataError:
-        If any metadata has expired.
+         If any of the top-level metadata is expired (whether a new version was
+         downloaded expired or no new version was found and the existing
+         version is now expired). 
 
     <Side Effects>
       Updates the metadata files of the top-level roles with the latest
@@ -643,6 +653,13 @@ def refresh(self, unsafely_update_root_if_necessary=True):
         logger.info('An expired Root metadata was loaded and must be updated.')
         raise
 
+    # If an exception is raised during the metadata update attempts, we will
+    # attempt to update root metadata once by recursing with a special argument
+    # to avoid further recursion. We use this bool and pull the recursion out
+    # of the except block so as to avoid unprintable nested NoWorkingMirrorError
+    # exceptions.
+    retry_once = False
+
     # Use default but sane information for timestamp metadata, and do not
     # require strict checks on its required length.
     try: 
@@ -652,19 +669,53 @@ def refresh(self, unsafely_update_root_if_necessary=True):
       self._update_metadata_if_changed('root')
       self._update_metadata_if_changed('targets')
 
+    # There are two distinct error scenarios that can rise from the
+    # _update_metadata_if_changed calls in the try block above:
+    #
+    #   - tuf.NoWorkingMirrorError:
+    #
+    #      If a change to a metadata file IS detected in an
+    #      _update_metadata_if_changed call, but we are unable to download a
+    #      valid (not expired, properly signed, valid) version of that metadata
+    #      file, and unexpired version of that metadata file, a
+    #      tuf.NoWorkingMirrorError rises to this point.
+    # 
+    #   - tuf.ExpiredMetadataError:
+    #
+    #      If, on the other hand, a change to a metadata file IS NOT detected in
+    #      a given _update_metadata_if_changed call, but we observe that the
+    #      version of the metadata file we have on hand is now expired, a
+    #      tuf.ExpiredMetadataError exception rises to this point.
+    #
     except tuf.NoWorkingMirrorError as e:
       if unsafely_update_root_if_necessary:
-        message = 'Valid top-level metadata cannot be downloaded.  Unsafely '+\
-          'update the Root metadata.'
-        logger.info(message)
-
-        self._update_metadata('root', DEFAULT_ROOT_UPPERLENGTH)
-        self.refresh(unsafely_update_root_if_necessary=False)
-
+        logger.info('Valid top-level metadata cannot be downloaded.  Unsafely '
+          'update the Root metadata.')
+        retry_once = True
+
       else:
         raise
 
+    except tuf.ExpiredMetadataError as e:
+      if unsafely_update_root_if_necessary:
+        logger.info('No changes were detected from the mirrors for a given role'
+          ', and that metadata that is available on disk has been found to be '
+          'expired. Trying to update root in case of foul play.')
+        retry_once = True
+
+      # The caller explicitly requested not to unsafely fetch an expired Root.
+      else:
+        logger.info('No changes were detected from the mirrors for a given role'
+          ', and that metadata that is available on disk has been found to be '
+          'expired. Your metadata is out of date.')
+        raise
 
+    # Update failed and we aren't already in a retry. Try once more after
+    # updating root.
+    if retry_once:
+        self._update_metadata('root', DEFAULT_ROOT_UPPERLENGTH)
+        self.refresh(unsafely_update_root_if_necessary=False)
+
 
 
 
@@ -1578,6 +1629,13 @@ def _update_metadata_if_changed(self, metadata_role,
                                          expected_versioninfo):
       logger.info(repr(uncompressed_metadata_filename) + ' up-to-date.')
 
+      # Since we have not downloaded a new version of this metadata, we
+      # should check to see if our version is stale and notify the user
+      # if so. This raises tuf.ExpiredMetadataError if the metadata we
+      # have is expired. Resolves issue #322.
+      self._ensure_not_expired(self.metadata['current'][metadata_role],
+                               metadata_role)
+
       return
 
     logger.debug('Metadata ' + repr(uncompressed_metadata_filename) + ' has changed.')