allowlist for source containers using lookaside cache

* CLOUDBLD-9969

Signed-off-by: Robert Cerven <rcerven@redhat.com>

atomic_reactor/plugins/pre_fetch_sources.py

@@ -8,10 +8,12 @@
 import os
 import shutil
 import tempfile
+import re
 
 import koji
 import tarfile
 import yaml
+from typing import List, Dict, Any
 
 from atomic_reactor.constants import (PLUGIN_FETCH_SOURCES_KEY, PNC_SYSTEM_USER,
                                       REMOTE_SOURCE_JSON_FILENAME, REMOTE_SOURCE_TARBALL_FILENAME,
@@ -186,6 +188,23 @@ def set_koji_image_build_data(self):
         if not self.koji_build_nvr:
             self.koji_build_nvr = self.koji_build['nvr']
 
+    def _get_cache_allowlist(self) -> List[Dict[str, Any]]:
+        src_config = get_source_container(self.workflow, fallback={})
+        allowlist_cache_url = src_config.get('lookaside_cache_allowlist')
+
+        if not allowlist_cache_url:
+            self.log.debug('no "lookaside_cache_allowlist" defined, '
+                           'not allowing any lookaside cache usage')
+            return []
+
+        self.log.debug('"lookaside_cache_allowlist" defined, might allow lookaside cache usage')
+        request_session = get_retrying_requests_session()
+        response = request_session.get(allowlist_cache_url)
+        response.raise_for_status()
+        allowlist_cache_yaml = yaml.safe_load(response.text)
+
+        return allowlist_cache_yaml
+
     def check_lookaside_cache_usage(self):
         """Check usage of lookaside cache, and fail if used"""
         git_uri, git_commit = self.koji_build['source'].split('#')
@@ -194,12 +213,44 @@ def check_lookaside_cache_usage(self):
         source_path = source.get()
         sources_cache_file = os.path.join(source_path, 'sources')
 
+        uses_cache = False
         if os.path.exists(sources_cache_file):
             if os.path.getsize(sources_cache_file) > 0:
-                raise RuntimeError('Repository is using lookaside cache, which is not allowed '
-                                   'for source container builds')
+                uses_cache = True
+
         source.remove_tmpdir()
 
+        if not uses_cache:
+            return
+
+        allowlist_cache_yaml = self._get_cache_allowlist()
+        should_raise = True
+
+        if allowlist_cache_yaml:
+            build_component = self.koji_build['package_name']
+            build_task = self.koji_build['extra']['container_koji_task_id']
+            task_info = self.session.getTaskInfo(build_task, request=True)
+            build_target = task_info['request'][1]
+
+            for allowed in allowlist_cache_yaml:
+                if allowed['component'] != build_component:
+                    continue
+
+                for target in allowed['targets']:
+                    if re.match(target, build_target):
+                        self.log.debug('target "%s" for component "%s" has allowed using '
+                                       'lookaside cache', build_target, build_component)
+
+                        should_raise = False
+                        break
+
+                if not should_raise:
+                    break
+
+        if should_raise:
+            raise RuntimeError('Repository is using lookaside cache, which is not allowed '
+                               'for source container builds')
+
     def assemble_srpm_url(self, base_url, srpm_filename, sign_key=None):
         """Assemble the URL used to fetch an SRPM file
 

atomic_reactor/schemas/config.json

@@ -695,6 +695,10 @@
               "description": "Url with denylist yaml file, which will be used to exclude sources from cachito ",
               "type": "string"
           },
+          "lookaside_cache_allowlist": {
+              "description": "Url with allowlist yaml file, which allows usage of lookaside cache",
+              "type": "string"
+          },
           "cpu_request": {
               "description": "Openshift cpu request for build",
               "type": "string",

tests/plugins/test_fetch_sources.py

@@ -62,10 +62,12 @@
                               'operator-manifests': {}},
                     'source': 'registry.com/repo#ref'}
 KOJI_BUILD_RS = {'build_id': 1, 'nvr': 'foobar-1-1', 'name': 'foobar', 'version': 1, 'release': 1,
+                 'package_name': 'foobar',
                  'extra': {'image': {'parent_build_id': 10,
                                      'pnc': {'builds': [{'id': 1234}]},
                                      'remote_source_url': 'remote_url'},
-                           'operator-manifests': {}},
+                           'operator-manifests': {},
+                           'container_koji_task_id': 1},
                  'source': 'registry.com/repo#ref'}
 KOJI_BUILD_MRS = {'build_id': 1, 'nvr': 'foobar-1-1', 'name': 'foobar', 'version': 1, 'release': 1,
                   'extra': {'image': {'parent_build_id': 10,
@@ -898,28 +900,89 @@ def test_denylist_srpms(self, requests_mock, docker_tasker, koji_session, tmpdir
         elif denylist_json and exc_str is None:
             assert 'denylisted srpms: ' in caplog.text
 
+    @pytest.mark.parametrize(('allowlist_url', 'allowlist_json', 'component',
+                              'target', 'in_list'), [
+        ('http://myallowlist',
+         [],
+         'foobar', 'target1', False),
+
+        ('http://myallowlist',
+         [{'component': 'foobar', 'targets': [r'target\d', r'\dtarget']}],
+         'foobar', 'target1', True),
+
+        ('http://myallowlist',
+         [{'component': 'foobar', 'targets': [r'notargets\d', r'\dnotargets']}],
+         'foobar', 'target1', False),
+
+        ('http://myallowlist',
+         [{'component': 'other', 'targets': [r'target\d', r'\dtarget']}],
+         'foobar', 'target1', False),
+
+        ('',
+         [],
+         'foobar', 'target1', False)])
     @pytest.mark.parametrize('use_cache', [True, False, None])
-    def test_lookaside_cache(self, requests_mock, docker_tasker, koji_session, tmpdir, use_cache):
+    def test_lookaside_cache(self, caplog, requests_mock, docker_tasker, koji_session, tmpdir,
+                             allowlist_url, allowlist_json, component, target, in_list, use_cache):
         mock_koji_manifest_download(tmpdir, requests_mock)
-        koji_build_nvr = 'foobar-1-1'
-        runner = mock_env(tmpdir, docker_tasker, koji_build_nvr=koji_build_nvr)
+        koji_build_nvr = f'{component}-1-1'
 
         if use_cache:
             tmpdir.join('sources').write('#ref file.tar.gz')
         elif use_cache is None:
             tmpdir.join('sources').write('')
 
+        rcm_json = yaml.safe_load(BASE_CONFIG_MAP)
+        rcm_json['source_container'] = {}
+
+        if allowlist_url:
+            rcm_json['source_container'] = {'lookaside_cache_allowlist': allowlist_url}
+
+        if allowlist_url and not allowlist_json:
+            requests_mock.register_uri('GET', allowlist_url,
+                                       reason='Not Found: {}'.format(allowlist_url),
+                                       status_code=404)
+
+        elif allowlist_url and allowlist_json:
+            requests_mock.register_uri('GET', allowlist_url,
+                                       json=allowlist_json, status_code=200)
+
+        task_json = {'request': ['repo', target]}
+        (flexmock(koji_session)
+         .should_receive('getTaskInfo')
+         .and_return(task_json))
+
+        runner = mock_env(tmpdir, docker_tasker, koji_build_nvr=koji_build_nvr,
+                          config_map=yaml.safe_dump(rcm_json))
+
         err_msg = 'Repository is using lookaside cache, which is not allowed ' \
                   'for source container builds'
+        if use_cache and allowlist_url and not allowlist_json:
+            err_msg = f'Not Found: {allowlist_url}'
 
-        if use_cache:
+        log_msg = f'target "{target}" for component "{component}" has allowed ' \
+                  f'using lookaside cache'
+
+        if use_cache and not in_list:
             with pytest.raises(PluginFailedException) as exc_info:
                 runner.run()
 
             assert err_msg in str(exc_info.value)
         else:
             runner.run()
 
+            if use_cache and in_list:
+                assert log_msg in caplog.text
+
+            if use_cache and not allowlist_url:
+                log_msg = 'no "lookaside_cache_allowlist" defined, ' \
+                          'not allowing any lookaside cache usage'
+                assert log_msg in caplog.text
+
+            if use_cache and allowlist_url:
+                log_msg = '"lookaside_cache_allowlist" defined, might allow lookaside cache usage'
+                assert log_msg in caplog.text
+
     @pytest.mark.parametrize('reason', ['external', 'other'])
     def test_missing_srpm_header(self, docker_tasker, koji_session, tmpdir, reason):
         (flexmock(koji_session)

tests/requirements.txt

@@ -9,3 +9,5 @@ pytest-html
 flake8
 koji
 requests-mock
+typing-extensions < 4.2.0;python_version<="3.6"
+typing-extensions;python_version>"3.6"