Merge pull request #532 from kids-first/py39

⬆️ Update dependencies for Python 3.9

.circleci/config.yml

@@ -3,44 +3,55 @@
 # Check https://circleci.com/docs/2.0/language-python/ for more details
 #
 version: 2
+
+test: &test
+  steps:
+    - checkout
+    - run: |
+        pip install -r dev-requirements.txt
+        pip install .
+        pytest tests --junit-xml test_output/junit.xml
+    - store_test_results:
+        path: test_output
+
 jobs:
-  build:
+  py37:
+    <<: *test
     docker:
-      # specify the version you desire here
-      # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
       - image: circleci/python:3.7
 
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      # - image: circleci/postgres:9.4
+  py38:
+    <<: *test
+    docker:
+      - image: circleci/python:3.8
 
-    working_directory: ~/repo
+  py39:
+    <<: *test
+    docker:
+      - image: circleci/python:3.9
 
+  lint:
+    docker:
+      - image: circleci/python:3.7
     steps:
       - checkout
+      - run: |
+          pip install black
+          black --check --line-length 80 kf_lib_data_ingest tests
+      - run: |
+          pip install flake8
+          flake8 --ignore=E501,W503,E203 kf_lib_data_ingest
+          flake8 --ignore=E501,W503,E203 tests
 
-      - run:
-          name: test package installation
-          command: |
-              python3 -m venv package_venv
-              . package_venv/bin/activate
-              pip install .
-              kidsfirst -h
-              rm -rf package_venv
-
-      - run:
-          name: build lib and docs
-          command: ./scripts/build.sh
-
-      - run:
-          name: run tests
-          command: ./scripts/test.sh
-
-      - store_artifacts:
-          path: test-reports
-          destination: test-reports
-
+  docs:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - run: |
+          pip install -r doc-requirements.txt
+          doc8 docs/source
+          make -C docs html
       - add_ssh_keys:
           fingerprints:
               - "64:fd:f9:62:bb:b3:48:56:a4:bb:17:9f:f8:e5:b8:7f"
@@ -50,3 +61,14 @@ jobs:
             if [ $CIRCLE_BRANCH == 'master' ]; then
                 ./scripts/deploy-ghpages.sh
             fi
+
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - py37
+      - py38
+      - py39
+      - lint
+      - docs

dev-requirements.txt

@@ -1,7 +1,7 @@
 pytest-cache==1.0
 pytest-cov==2.10.1
 codacy-coverage==1.3.11
-moto==1.3.14
+moto==1.3.16
 requests-mock==1.8.0
 flake8
 black

kf_lib_data_ingest/common/constants.py

@@ -151,7 +151,7 @@ class CENTER:
         class ASHION:
             NAME = "Ashion"
             KF_ID = "SC_0CNMF82N"
-            
+
         class BAYLOR:
             NAME = "Baylor College of Medicine"
             KF_ID = "SC_A1JNZAZH"

requirements.txt

@@ -1,15 +1,16 @@
 click==7.1.2
 jsonschema==3.2.0
-pandas==1.1.1
+pandas==1.1.4
 PyYAML==5.3.1
-boto3==1.14.52
-xlrd==1.2.0
+boto3==1.16.21
+xlrd @ git+https://github.com/fiendish/xlrd.git
+openpyxl
 tabulate==0.8.7
 sqlite3worker==1.1.7
 jsonpickle==1.4.1
-SQLAlchemy==1.3.19
-pytest==6.0.1
+SQLAlchemy==1.3.20
+pytest==6.1.2
 cchardet>=2.1.6<3
-graph_theory==2020.10.7.47043
+graph_theory==2020.11.4.41115
 d3b_utils @ git+https://github.com/d3b-center/d3b-utils-python.git
 kf_utils @ git+https://github.com/kids-first/kf-utils-python.git

tests/test_file_retriever.py

@@ -51,13 +51,17 @@ def s3_file():
     """
     mock_s3().start()
     # Create bucket
+    location = "us-west-2"
     client = boto3.client(
         "s3",
-        region_name="eu-west-1",
+        region_name=location,
         aws_access_key_id="fake_access_key",
         aws_secret_access_key="fake_secret_key",
     )
-    client.create_bucket(Bucket=TEST_S3_BUCKET)
+    client.create_bucket(
+        Bucket=TEST_S3_BUCKET,
+        CreateBucketConfiguration={"LocationConstraint": location},
+    )
 
     # Upload file
     client.upload_file(