Switch to pymediawiki

Signed-off-by: Mattia Verga <mattia.verga@tiscali.it>

bodhi-server/bodhi/server/config.py

@@ -617,6 +617,9 @@ class BodhiConfig(dict):
         'wiki_url': {
             'value': 'https://fedoraproject.org/w/api.php',
             'validator': str},
+        'wiki_user_agent': {
+            'value': 'FedoraLinuxBodhi-User-Agent',
+            'validator': str},
     }
 
     def __getitem__(self, key: typing.Hashable) -> typing.Any:

bodhi-server/bodhi/server/models.py

@@ -29,8 +29,8 @@
 import typing
 import uuid
 
+from mediawiki import MediaWiki
 from packaging.version import parse as parse_version
-from simplemediawiki import MediaWiki
 from sqlalchemy import __version__ as sqlalchemy_version
 from sqlalchemy import (
     and_,
@@ -1592,30 +1592,26 @@ def update_test_cases(self, db):
 
         start = datetime.utcnow()
         log.debug(f'Querying the wiki for test cases of {self.nvr}')
-
-        wiki = MediaWiki(config.get('wiki_url'))
-        cat_page = f'Category:Package {self.package.external_name} test cases'
+        try:
+            wiki = MediaWiki(config.get('wiki_url'),
+                             user_agent=config.get('wiki_user_agent'))
+        except Exception as ex:
+            raise ExternalCallException(f'Failed to connect to Fedora Wiki: {ex}')
+        cat_page = f'Package {self.package.external_name} test cases'
 
         def list_categorymembers(wiki, cat_page, limit=500):
-            # Build query arguments and call wiki
-            query = dict(action='query', list='categorymembers',
-                         cmtitle=cat_page, cmlimit=limit)
             try:
-                response = wiki.call(query)
-            except URLError:
-                raise ExternalCallException('Failed retrieving testcases from Wiki')
-            members = [entry['title'] for entry in
-                       response.get('query', {}).get('categorymembers', {})
-                       if 'title' in entry]
+                response = wiki.categorymembers(cat_page, results=limit, subcategories=True)
+            except Exception as ex:
+                raise ExternalCallException(f'Failed retrieving testcases from Wiki: {ex}')
+            members = [entry for entry in response[0] if entry != '']
 
             # Determine whether we need to recurse
-            if limit > 0:
-                for idx, member in enumerate(copy(members)):
-                    if member.startswith('Category:'):
-                        members.extend(list_categorymembers(wiki, member, limit - 1))
-                        members.remove(members[idx])  # remove Category from list
+            if len(response[1]) > 0 and len(members) < limit:
+                for subcat in response[1]:
+                    members.extend(list_categorymembers(wiki, subcat, limit=limit - len(members)))
 
-            log.debug(f'Found the following unit tests: {members}')
+            log.debug(f'Found the following testcases: {members}')
             return members
 
         fetched = set(list_categorymembers(wiki, cat_page))

bodhi-server/pyproject.toml

@@ -102,13 +102,13 @@ prometheus-client = ">=0.13.1"
 psycopg2 = ">=2.8.6"
 py3dns = ">=3.2.1" # Should be required by pyLibravatar https://bugs.launchpad.net/pylibravatar/+bug/1661218
 pyLibravatar = ">=1.6"
+pymediawiki = "^0.7"
 pyramid = ">=1.10.5"
 pyramid-fas-openid = ">=0.4.0"
 pyramid-mako = ">=1.0.2"
 python-bugzilla = ">=3.2.0"
 PyYAML = ">=5.4.1"
 requests = ">=2.25.1"
-simplemediawiki = {version = "^1.2.0b2", allow-prereleases = true}
 SQLAlchemy = ">=1.3.24"
 waitress = ">=1.4.4"
 

bodhi-server/tests/tasks/test_fetch_test_cases.py

@@ -20,8 +20,8 @@
 """
 
 from unittest.mock import patch
-from urllib.error import URLError
 
+from mediawiki.exceptions import HTTPTimeoutError
 import pytest
 
 from bodhi.server import config, models
@@ -70,7 +70,7 @@ def test_fetch_test_cases_exception(self, warning, MediaWiki):
         """
         Assert that fetch_test_cases logs a warning when an exception is raised.
         """
-        MediaWiki.return_value.call.side_effect = URLError("oh no!")
+        MediaWiki.return_value.categorymembers.side_effect = HTTPTimeoutError("oh no!")
 
         update = self.db.query(models.Update).join(models.Build).filter(
             models.Build.nvr == 'bodhi-2.0-1.fc17').one()

bodhi-server/tests/test_models.py

@@ -18,7 +18,6 @@
 """Test suite for bodhi.server.models"""
 from datetime import datetime, timedelta
 from unittest import mock
-from urllib.error import URLError
 import hashlib
 import html
 import json
@@ -28,6 +27,7 @@
 
 from fedora_messaging.api import Message
 from fedora_messaging.testing import mock_sends
+from mediawiki.exceptions import HTTPTimeoutError, MediaWikiAPIURLError
 from pyramid.testing import DummyRequest
 from sqlalchemy.exc import IntegrityError
 import cornice
@@ -1009,20 +1009,6 @@ def test_clear_all_releases_cache(self):
         assert model.Release._all_releases is None
 
 
-class MockWiki(object):
-    """ Mocked simplemediawiki.MediaWiki class. """
-    def __init__(self, response):
-        self.response = response
-        self.query = None
-
-    def __call__(self, *args, **kwargs):
-        return self
-
-    def call(self, query):
-        self.query = query
-        return self.response
-
-
 class TestPackageModel(BasePyTestCase):
     """Tests for the Package model."""
 
@@ -1639,40 +1625,29 @@ def do_get_dependencies(self):
         return {'package': model.Package(name='TurboGears')}
 
     @mock.patch.dict(config, {'query_wiki_test_cases': True})
-    def test_wiki_test_cases(self):
+    @mock.patch('bodhi.server.models.MediaWiki')
+    def test_wiki_test_cases(self, MediaWiki):
         """Test querying the wiki for test cases"""
-        # Mock out mediawiki so we don't do network calls in our tests
-        response = {
-            'query': {
-                'categorymembers': [{
-                    'title': 'Fake test case',
-                }],
-            }
-        }
-
-        # Now, our actual test.
-        with mock.patch('bodhi.server.models.MediaWiki', MockWiki(response)):
-            pkg = model.RpmPackage(name='gnome-shell')
-            self.db.add(pkg)
-            build = model.RpmBuild(nvr='gnome-shell-1.1.1-1.fc32', package=pkg)
-            self.db.add(build)
-            build.update_test_cases(self.db)
-            assert build.testcases[0].name == 'Fake test case'
-            assert len(build.testcases) == 1
+        responses = [
+            (['Fake test case'], [])]
+        MediaWiki.return_value.categorymembers.side_effect = responses
+        pkg = model.RpmPackage(name='gnome-shell')
+        self.db.add(pkg)
+        build = model.RpmBuild(nvr='gnome-shell-1.1.1-1.fc32', package=pkg)
+        self.db.add(build)
+        build.update_test_cases(self.db)
+        assert model.TestCase.query.count() == 1
+        assert build.testcases[0].name == 'Fake test case'
+        assert len(build.testcases) == 1
 
     @mock.patch.dict(config, {'query_wiki_test_cases': True})
     @mock.patch('bodhi.server.models.MediaWiki')
     def test_wiki_test_cases_recursive(self, MediaWiki):
         """Test querying the wiki for test cases when recursion is necessary."""
         responses = [
-            {'query': {
-                'categorymembers': [
-                    {'title': 'Fake'},
-                    {'title': 'Category:Bodhi'},
-                    {'title': 'Uploading cat pictures'}]}},
-            {'query': {
-                'categorymembers': [
-                    {'title': 'Does Bodhi eat +1s'}]}}]
+            (['Fake', 'Uploading cat pictures'], ['Bodhi']),
+            (['Does Bodhi eat +1s'], [])]
+        MediaWiki.return_value.categorymembers.side_effect = responses
         MediaWiki.return_value.call.side_effect = responses
         pkg = model.RpmPackage(name='gnome-shell')
         self.db.add(pkg)
@@ -1691,18 +1666,15 @@ def test_wiki_test_cases_recursive(self, MediaWiki):
     def test_wiki_test_cases_removed(self, MediaWiki):
         """Test querying the wiki for test cases and remove test which aren't actual."""
         responses = [
-            {'query': {
-                'categorymembers': [
-                    {'title': 'Fake test case'},
-                    {'title': 'Does Bodhi eat +1s'}]}}]
+            (['Fake test case', 'Does Bodhi eat +1s'], [])]
 
         pkg = model.RpmPackage(name='gnome-shell')
         self.db.add(pkg)
         build = model.RpmBuild(nvr='gnome-shell-1.1.1-1.fc32', package=pkg)
         self.db.add(build)
 
         # Add both tests to build
-        MediaWiki.return_value.call.side_effect = responses
+        MediaWiki.return_value.categorymembers.side_effect = responses
         build.update_test_cases(self.db)
         assert model.TestCase.query.count() == 2
         assert len(build.testcases) == 2
@@ -1711,20 +1683,33 @@ def test_wiki_test_cases_removed(self, MediaWiki):
 
         # Now remove one test
         responses = [
-            {'query': {
-                'categorymembers': [
-                    {'title': 'Fake test case'}]}}]
-        MediaWiki.return_value.call.side_effect = responses
+           (['Fake test case'], [])]
+        MediaWiki.return_value.categorymembers.side_effect = responses
         build.update_test_cases(self.db)
         assert model.TestCase.query.count() == 2
         assert len(build.testcases) == 1
         assert build.testcases[0].name == 'Fake test case'
 
     @mock.patch.dict(config, {'query_wiki_test_cases': True})
     @mock.patch('bodhi.server.models.MediaWiki')
-    def test_wiki_test_cases_exception(self, MediaWiki):
+    def test_wiki_connection_failed(self, MediaWiki):
+        """Test querying the wiki for test cases when connection to Wiki failed"""
+        MediaWiki.side_effect = MediaWikiAPIURLError("https://bad-api-url")
+
+        with pytest.raises(ExternalCallException) as exc_context:
+            pkg = model.RpmPackage(name='gnome-shell')
+            self.db.add(pkg)
+            build = model.RpmBuild(nvr='gnome-shell-1.1.1-1.fc32', package=pkg)
+            self.db.add(build)
+            build.update_test_cases(self.db)
+        assert len(build.testcases) == 0
+        assert str(exc_context.value).startswith('Failed to connect to Fedora Wiki:')
+
+    @mock.patch.dict(config, {'query_wiki_test_cases': True})
+    @mock.patch('bodhi.server.models.MediaWiki')
+    def test_wiki_query_timeout(self, MediaWiki):
         """Test querying the wiki for test cases when connection to Wiki failed"""
-        MediaWiki.return_value.call.side_effect = URLError("oh no!")
+        MediaWiki.return_value.categorymembers.side_effect = HTTPTimeoutError("oh no!")
 
         with pytest.raises(ExternalCallException) as exc_context:
             pkg = model.RpmPackage(name='gnome-shell')
@@ -1733,7 +1718,7 @@ def test_wiki_test_cases_exception(self, MediaWiki):
             self.db.add(build)
             build.update_test_cases(self.db)
         assert len(build.testcases) == 0
-        assert str(exc_context.value) == 'Failed retrieving testcases from Wiki'
+        assert str(exc_context.value).startswith('Failed retrieving testcases from Wiki:')
 
 
 class TestRpmBuild(ModelTest):

devel/ansible/roles/bodhi/tasks/main.yml

@@ -58,6 +58,7 @@
       - python3-psycopg2
       - python3-pydocstyle
       - python3-pylibravatar
+      - python3-pymediawiki
       - python3-pyramid
       - python3-pyramid-fas-openid
       - python3-pytest
@@ -66,7 +67,6 @@
       - python3-responses
       - python3-rpdb
       - python3-setuptools
-      - python3-simplemediawiki
       - python3-sphinx
       - python3-sqlalchemy
       - python3-sqlalchemy_schemadisplay

devel/ci/Dockerfile-f37

@@ -41,6 +41,7 @@ RUN dnf install -y \
     python3-psycopg2 \
     python3-prometheus_client \
     python3-pylibravatar \
+    python3-pymediawiki \
     python3-pyramid \
     python3-pyramid-fas-openid \
     python3-pyramid-mako \
@@ -49,7 +50,6 @@ RUN dnf install -y \
     python3-pytest-mock \
     python3-requests-kerberos \
     python3-responses \
-    python3-simplemediawiki \
     python3-sphinx \
     python3-sqlalchemy \
     python3-sqlalchemy_schemadisplay \

devel/ci/Dockerfile-pip

@@ -17,7 +17,6 @@ RUN dnf install -y \
     postgresql-devel \
     python3-devel \
     python3-librepo \
-    python3-simplemediawiki \
     redhat-rpm-config \
     python3-libcomps \
     python3-gssapi \
@@ -28,6 +27,7 @@ RUN dnf install -y \
 
 RUN pip-3 install \
     pyasn1 \
+    pymediawiki \
     celery
 
 VOLUME ["/results"]

devel/ci/Dockerfile-rawhide

@@ -41,6 +41,7 @@ RUN dnf install -y \
     python3-psycopg2 \
     python3-prometheus_client \
     python3-pylibravatar \
+    python3-pymediawiki \
     python3-pyramid \
     python3-pyramid-fas-openid \
     python3-pyramid-mako \
@@ -49,7 +50,6 @@ RUN dnf install -y \
     python3-pytest-mock \
     python3-requests-kerberos \
     python3-responses \
-    python3-simplemediawiki \
     python3-sphinx \
     python3-sqlalchemy \
     python3-sqlalchemy_schemadisplay \