Add isort to CI and pre-commit configuration

.github/workflows/static.yml

@@ -47,6 +47,10 @@ jobs:
         run: |
           poetry run black --diff --check .
 
+      - name: Import-check with isort
+        run: |
+          poetry run isort --diff --check-only .
+
       - name: Lint with flake8
         run: |
           poetry run flake8 --show-source --statistics .

.pre-commit-config.yaml

@@ -9,6 +9,16 @@ repos:
         files: ^src/|^tests/
         types: [python]
 
+  - repo: local
+    hooks:
+      - id: python-import
+        name: isort
+        entry: poetry run isort
+        args: [--diff, --check-only]
+        language: system
+        files: ^src/|^tests/
+        types: [python]
+
   - repo: local
     hooks:
       - id: python-lint

pyproject.toml

@@ -28,6 +28,7 @@ flake8 = "^5.0.4"
 shellcheck-py = "^0.8.0"
 shfmt-py = "^3.4.3"
 pytest-mock = "^3.8.2"
+isort = "^5.10.1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -48,6 +49,12 @@ extend-exclude = """
 )/
 """
 
+[tool.isort]
+profile = "black"
+skip_glob = ["data/*", "runs/*", "scripts/*",
+             "submodules/*", ".venv/*", "cov_html/*"]
+line_length = 88
+
 [tool.coverage.run]
 branch = true
 source = ["src"]

requirements-dev.txt

@@ -19,14 +19,15 @@ evaluate==0.2.2; python_full_version >= "3.7.0"
 filelock==3.4.0; python_version >= "3.6" and python_full_version >= "3.7.0"
 flake8==5.0.4; python_full_version >= "3.6.1"
 frozenlist==1.2.0; python_version >= "3.6"
-fsspec==2022.8.2; python_version >= "3.7" and python_full_version >= "3.7.0"
+fsspec==2022.10.0; python_version >= "3.7" and python_full_version >= "3.7.0"
 gitdb==4.0.9; python_version >= "3.7"
 gitpython==3.1.27; python_version >= "3.7"
 huggingface-hub==0.9.1; python_full_version >= "3.7.0"
 idna==3.3; python_version >= "3.6" and python_full_version >= "3.7.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 iniconfig==1.1.1; python_version >= "3.7"
 ipdb==0.13.9; python_version >= "2.7"
 ipython==7.30.0; python_version >= "3.7"
+isort==5.10.1; python_full_version >= "3.6.1" and python_version < "4.0"
 jedi==0.18.1; python_version >= "3.7"
 joblib==1.1.0; python_version >= "3.8"
 matplotlib-inline==0.1.3; python_version >= "3.7"

requirements.txt

@@ -12,7 +12,7 @@ docker-pycreds==0.4.0; python_version >= "3.6"
 evaluate==0.2.2; python_full_version >= "3.7.0"
 filelock==3.4.0; python_version >= "3.6" and python_full_version >= "3.7.0"
 frozenlist==1.2.0; python_version >= "3.6"
-fsspec==2022.8.2; python_version >= "3.7" and python_full_version >= "3.7.0"
+fsspec==2022.10.0; python_version >= "3.7" and python_full_version >= "3.7.0"
 gitdb==4.0.9; python_version >= "3.7"
 gitpython==3.1.27; python_version >= "3.7"
 huggingface-hub==0.9.1; python_full_version >= "3.7.0"

src/parser.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from transformers.hf_argparser import DataClassType
-from transformers import HfArgumentParser, TrainingArguments
+import os
 from dataclasses import dataclass, field
 from typing import Optional
-import os
+
+from transformers import HfArgumentParser, TrainingArguments
+from transformers.hf_argparser import DataClassType
 
 # 1. NOTE: Fields with no default value set will be transformed
 # into`required arguments within the HuggingFace argument parser

src/privacy_glue.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from parser import get_parser
+
 from utils.experiment_utils import Privacy_GLUE_Experiment_Manager
 
 

src/tasks/opp_115.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import os
+
 import datasets
 import pandas as pd
-import os
 
 
 def load_opp_115(directory: str) -> datasets.DatasetDict:

src/tasks/piextract.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from typing import Dict, List, Tuple
+import os
 from glob import glob
+from typing import Dict, List, Tuple
+
 import datasets
-import os
 
 
 def read_conll_file(file_path: str) -> Dict[str, List[List[str]]]:

src/tasks/policy_detection.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import pandas as pd
-import datasets
 import os
 
+import datasets
+import pandas as pd
+
 
 def load_policy_detection(directory: str) -> datasets.DatasetDict:
     # initialize DatasetDict object

src/tasks/policy_ie_a.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import os
 from typing import Dict, cast
+
 import datasets
-import os
 
 
 def file_mapping(directory: str, filename: str) -> Dict[str, str]:

src/tasks/policy_ie_b.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from .policy_ie_a import file_mapping
 from typing import cast
+
 import datasets
 
+from .policy_ie_a import file_mapping
+
 
 def load_policy_ie_b(directory: str) -> datasets.DatasetDict:
     # initialize DatasetDict object

src/tasks/policy_qa.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from typing import Dict, List, Any
-from glob import glob
-import datasets
 import json
 import os
+from glob import glob
+from typing import Any, Dict, List
+
+import datasets
 
 
 def load_policy_qa(directory: str) -> datasets.DatasetDict:

src/tasks/privacy_qa.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import pandas as pd
-import datasets
 import os
 
+import datasets
+import pandas as pd
+
 
 def load_privacy_qa(directory: str) -> datasets.DatasetDict:
     # load and process the train dataset

src/utils/experiment_utils.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import os
+import re
+from parser import TASKS
+
 from wandb.util import generate_id
+
+from reading_comprehension import Reading_Comprehension_Pipeline
 from sequence_classification import Sequence_Classification_Pipeline
 from sequence_tagging import Sequence_Tagging_Pipeline
-from reading_comprehension import Reading_Comprehension_Pipeline
-from parser import TASKS
-import os
-import re
 
 
 class Privacy_GLUE_Experiment_Manager:

src/utils/parser_utils.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from typing import Iterable, Optional, cast
-from operator import attrgetter
 import argparse
 import os
 import re
+from operator import attrgetter
+from typing import Iterable, Optional, cast
 
 
 def dir_path(path: str) -> str:

src/utils/pipeline_utils.py

@@ -1,27 +1,29 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import logging
+import os
+import shutil
 from abc import ABC, abstractmethod
-from datasets import DatasetDict
 from glob import glob
+from parser import DataArguments, ModelArguments
+
+import datasets
+import torch
+import transformers
+import wandb
+from datasets import DatasetDict
 from transformers import TrainingArguments, set_seed
 from transformers.trainer_utils import get_last_checkpoint
-from parser import DataArguments, ModelArguments
+
 from tasks.opp_115 import load_opp_115
 from tasks.piextract import load_piextract
 from tasks.policy_detection import load_policy_detection
 from tasks.policy_ie_a import load_policy_ie_a
 from tasks.policy_ie_b import load_policy_ie_b
 from tasks.policy_qa import load_policy_qa
 from tasks.privacy_qa import load_privacy_qa
-from utils.logging_utils import init_logger, add_file_handler
-import transformers
-import datasets
-import logging
-import wandb
-import shutil
-import torch
-import os
+from utils.logging_utils import add_file_handler, init_logger
 
 
 class SuccessFileFoundException(Exception):

tests/conftest.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from types import SimpleNamespace
+import logging
+import os
+from copy import deepcopy
 from functools import partial
+from types import SimpleNamespace
 from unittest.mock import MagicMock
-from copy import deepcopy
+
 import datasets
-import logging
 import pytest
-import os
 
 
 def pytest_configure():

tests/tasks/test_opp_115.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from tasks.opp_115 import load_opp_115
 import os
 
+from tasks.opp_115 import load_opp_115
+
 
 def test_load_opp_115():
     # load sample data