fix: standardize requirements and fixes for marshmallow3+ (#98)

.github/CODEOWNERS

@@ -10,4 +10,4 @@
 # review when someone opens a pull request.
 *       @amundsen-io/amundsen-committers
 
-*.py    @feng-tao @jinhyukchang @allisonsuarez @dikshathakur3119 @verdan @bolkedebruin @mgorsk1
+*.py    @feng-tao @jinhyukchang @allisonsuarez @verdan @bolkedebruin @mgorsk1

.github/workflows/pull_request.yml

@@ -25,6 +25,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        run: pip3 install -r requirements.txt && pip3 install codecov
+        run: pip3 install -r requirements.txt && pip3 install codecov && pip install .
       - name: Run python unit tests
         run: make test

Makefile

@@ -1,9 +1,9 @@
- clean:
+clean:
 	find . -name \*.pyc -delete
 	find . -name __pycache__ -delete
 	rm -rf dist/
 
- test_unit:
+test_unit:
 	python -m pytest tests
 	python3 -bb -m pytest tests
 
@@ -16,4 +16,4 @@ mypy:
 	mypy .
 
 
- test: test_unit lint mypy
+test: test_unit lint mypy

amundsen_common/log/http_header_caller_retrieval.py

@@ -12,4 +12,4 @@
 class HttpHeaderCallerRetrieval(BaseCallerRetriever):
     def get_caller(self) -> str:
         header_key = flask_app.config.get(CALLER_HEADER_KEY, 'user-agent')
-        return request.headers.get(header_key, 'UNKNOWN')
+        return request.headers.get(header_key) or 'UNKNOWN'

amundsen_common/models/user.py

@@ -57,7 +57,7 @@ def _str_no_value(self, s: Optional[str]) -> bool:
         return False
 
     @pre_load
-    def preprocess_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
+    def preprocess_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
         if self._str_no_value(data.get('user_id')):
             data['user_id'] = data.get('email')
 
@@ -81,7 +81,7 @@ def preprocess_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
         return data
 
     @validates_schema
-    def validate_user(self, data: Dict[str, Any]) -> None:
+    def validate_user(self, data: Dict[str, Any], **kwargs: Any) -> None:
         if self._str_no_value(data.get('display_name')):
             raise ValidationError('"display_name", "full_name", or "email" must be provided')
 

requirements.txt

@@ -1,9 +1,5 @@
-attrs==19.1.0
 flake8==3.7.8
-Flask==1.1.1
-marshmallow==3.6.0
-git+https://www.github.com/hilearn/marshmallow-annotations.git@a7a2dc96932430369bdef36555082df990ed9bef#egg=marshmallow-annotations
-mypy==0.761
+mypy==0.812
 pytest>=4.6
 pytest-cov
 pytest-mock

setup.py

@@ -3,7 +3,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '0.8.1'
+__version__ = '0.8.2'
 
 setup(
     name='amundsen-common',
@@ -15,6 +15,10 @@
     maintainer='Amundsen TSC',
     maintainer_email='amundsen-tsc@lists.lfai.foundation',
     packages=find_packages(exclude=['tests*']),
+    # pip ignores dependency_links and only installs the `marshmallow-annotations @ ...` requirement
+    # specified in install_requires. Unfortunately easy_install which is invoked by `python setup.py install`
+    # does the opposite. For this reason we include this private fork in both sections so both easy_install and
+    # pip pick this up.
     dependency_links=[
         ('git+https://www.github.com/hilearn/marshmallow-annotations.git@a7a2dc96932430369bd'
          'ef36555082df990ed9bef#egg=marshmallow-annotations')
@@ -35,9 +39,10 @@
         # This will allow for any consuming projects to use this library as
         # long as they have a version of pyfoobar equal to or greater than 1.x
         # and less than 2.x installed.
-        'flask>=1.0.2',
+        'Flask>=1.0.2',
+        'attrs>=19.0.0',
         'marshmallow>=3.0,<=3.6',
-        'marshmallow-annotations'
+        'marshmallow-annotations @ git+https://www.github.com/hilearn/marshmallow-annotations.git@a7a2dc96932430369bdef36555082df990ed9bef#egg=marshmallow-annotations'  # noqa
     ],
     python_requires=">=3.6",
     package_data={'amundsen_common': ['py.typed']},

tests/unit/log/test_http_header_caller_retrieval.py

@@ -5,6 +5,7 @@
 
 import flask
 from mock import patch
+from mock import MagicMock
 
 from amundsen_common.log import http_header_caller_retrieval
 from amundsen_common.log.http_header_caller_retrieval import HttpHeaderCallerRetrieval
@@ -13,12 +14,11 @@
 
 
 class ActionLogTest(unittest.TestCase):
-
     def test(self) -> None:
-        with app.test_request_context(), patch.object(http_header_caller_retrieval, 'request') as mock_request:
+        with app.test_request_context(), \
+                patch.object(http_header_caller_retrieval, 'request', new=MagicMock()) as mock_request:
             mock_request.headers.get.return_value = 'foo'
             actual = HttpHeaderCallerRetrieval().get_caller()
-
             self.assertEqual(actual, 'foo')