support verify flag in requests (#17)

integration_tests/base_test.py

@@ -45,6 +45,7 @@ class BaseIntegrationTest(DependencyTestMixin, CleanRepoMixin):
     output_path = "test-codetf.txt"
     num_changes = 1
     _lines = []
+    num_changed_files = 1
 
     def _assert_run_fields(self, run, output_path):
         assert run["vendor"] == "pixee"
@@ -62,8 +63,13 @@ def _assert_results_fields(self, results, output_path):
         assert len(results) == 1
         result = results[0]
         assert result["codemod"] == self.codemod.full_name()
-        assert len(result["changeset"]) == 1
-        change = result["changeset"][0]
+        assert len(result["changeset"]) == self.num_changed_files
+
+        # A codemod may change multiple files. For now we will
+        # assert the resulting data for one file only.
+        change = [
+            result for result in result["changeset"] if result["path"] == output_path
+        ][0]
         assert change["path"] == output_path
         assert change["diff"] == self.expected_diff
 

integration_tests/semgrep/test_semgrep.py

@@ -41,6 +41,7 @@ def test_two_codemods(self):
         assert sorted(results_by_path_and_id.keys()) == [
             "tests/samples/insecure_random.py",
             "tests/samples/make_request.py",
+            "tests/samples/unverified_request.py",
         ]
 
         url_sandbox_results = results_by_path_and_id["tests/samples/make_request.py"][

integration_tests/test_request_verify.py

@@ -0,0 +1,24 @@
+from codemodder.codemods.requests_verify import RequestsVerify
+from integration_tests.base_test import (
+    BaseIntegrationTest,
+    original_and_expected_from_code_path,
+)
+
+
+class TestRequestsVerify(BaseIntegrationTest):
+    codemod = RequestsVerify
+    code_path = "tests/samples/unverified_request.py"
+    original_code, expected_new_code = original_and_expected_from_code_path(
+        code_path,
+        [
+            (2, """requests.get("www.google.com", verify=True)\n"""),
+            (
+                3,
+                """requests.post("https/some-api/", json={"id": 1234, "price": 18}, verify=True)\n""",
+            ),
+        ],
+    )
+    expected_diff = '--- \n+++ \n@@ -1,5 +1,5 @@\n import requests\n \n-requests.get("www.google.com", verify=False)\n-requests.post("https/some-api/", json={"id": 1234, "price": 18}, verify=False)\n+requests.get("www.google.com", verify=True)\n+requests.post("https/some-api/", json={"id": 1234, "price": 18}, verify=True)\n var = "hello"\n'
+    expected_line_change = "3"
+    num_changes = 2
+    change_description = RequestsVerify.CHANGE_DESCRIPTION

integration_tests/test_url_sandbox.py

@@ -19,6 +19,7 @@ class TestUrlSandbox(BaseIntegrationTest):
     expected_diff = '--- \n+++ \n@@ -1,4 +1,4 @@\n-import requests\n+from security import safe_requests\n \n-requests.get("www.google.com")\n+safe_requests.get("www.google.com")\n var = "hello"\n'
     expected_line_change = "3"
     change_description = UrlSandbox.CHANGE_DESCRIPTION
+    num_changed_files = 2
 
     requirements_path = "tests/samples/requirements.txt"
     original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"

src/codemodder/__main__.py

@@ -76,7 +76,7 @@ def run_codemods_for_file(
                 ChangeSet(
                     str(file_context.file_path),
                     diff,
-                    changes=codemod_kls.CHANGES_IN_FILE,
+                    changes=file_context.codemod_changes,
                 ).to_json()
             )
             if file_context.dry_run:

src/codemodder/codemods/__init__.py

@@ -13,6 +13,7 @@
 from codemodder.codemods.process_creation_sandbox import ProcessSandbox
 from codemodder.codemods.remove_unnecessary_f_str import RemoveUnnecessaryFStr
 from codemodder.codemods.tempfile_mktemp import TempfileMktemp
+from codemodder.codemods.requests_verify import RequestsVerify
 
 DEFAULT_CODEMODS = {
     DjangoDebugFlagOn,
@@ -28,6 +29,7 @@
     UpgradeSSLContextTLS,
     UrlSandbox,
     TempfileMktemp,
+    RequestsVerify,
 }
 ALL_CODEMODS = DEFAULT_CODEMODS
 

src/codemodder/codemods/api/__init__.py

@@ -70,11 +70,10 @@ class BaseCodemod(
     Helpers,
 ):
     CHANGESET_ALL_FILES: list = []
-    CHANGES_IN_FILE: list = []
 
     def report_change(self, original_node):
         line_number = self.lineno_for_node(original_node)
-        self.CHANGES_IN_FILE.append(
+        self.file_context.codemod_changes.append(
             Change(str(line_number), self.CHANGE_DESCRIPTION).to_json()
         )
 

src/codemodder/codemods/base_codemod.py

@@ -26,7 +26,6 @@ class BaseCodemod:
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
         cls.CHANGESET_ALL_FILES: list = []
-        cls.CHANGES_IN_FILE: list = []
 
         if "codemodder.codemods.base_codemod.SemgrepCodemod" in str(cls):
             # hack: SemgrepCodemod won't NotImplementedError but all other child

src/codemodder/codemods/django_debug_flag_on.py

@@ -38,7 +38,9 @@ def transform_module_impl(self, tree: cst.Module) -> cst.Module:
             )
             new_tree = debug_flag_transformer.transform_module(tree)
             if debug_flag_transformer.changes_in_file:
-                self.CHANGES_IN_FILE.extend(debug_flag_transformer.changes_in_file)
+                self.file_context.codemod_changes.extend(
+                    debug_flag_transformer.changes_in_file
+                )
                 return new_tree
         return tree
 

src/codemodder/codemods/django_session_cookie_secure_off.py

@@ -37,7 +37,7 @@ def transform_module_impl(self, tree: cst.Module) -> cst.Module:
             )
             new_tree = transformer.transform_module(tree)
             if transformer.changes_in_file:
-                self.CHANGES_IN_FILE.extend(transformer.changes_in_file)
+                self.file_context.codemod_changes.extend(transformer.changes_in_file)
             return new_tree
         return tree
 

src/codemodder/codemods/order_imports.py

@@ -54,7 +54,7 @@ def transform_module_impl(self, tree: cst.Module) -> cst.Module:
                     line_number = self.node_position(
                         top_imports_visitor.top_imports_blocks[i][0]
                     ).start.line
-                    self.CHANGES_IN_FILE.append(
+                    self.file_context.codemod_changes.append(
                         Change(str(line_number), self.CHANGE_DESCRIPTION).to_json()
                     )
             return result_tree

src/codemodder/codemods/remove_unused_imports.py

@@ -38,7 +38,7 @@ def transform_module_impl(self, tree: cst.Module) -> cst.Module:
         for import_alias, importt in gather_unused_visitor.unused_imports:
             pos = self.get_metadata(PositionProvider, import_alias)
             if self.filter_by_path_includes_or_excludes(pos):
-                RemoveUnusedImports.CHANGES_IN_FILE.append(
+                self.file_context.codemod_changes.append(
                     Change(pos.start.line, self.CHANGE_DESCRIPTION).to_json()
                 )
                 filtered_unused_imports.add((import_alias, importt))

src/codemodder/codemods/requests_verify.py

@@ -0,0 +1,25 @@
+from codemodder.codemods.base_codemod import ReviewGuidance
+from codemodder.codemods.api import SemgrepCodemod
+
+
+class RequestsVerify(SemgrepCodemod):
+    NAME = "requests-verify"
+    REVIEW_GUIDANCE = ReviewGuidance.MERGE_AFTER_CURSORY_REVIEW
+    DESCRIPTION = (
+        "Makes any calls to requests.{func} with `verify=False` to `verify=True`"
+    )
+
+    @classmethod
+    def rule(cls):
+        return """
+        rules:
+          - patterns:
+            - pattern: requests.$F(..., verify=False, ...)
+            - pattern-inside: |
+                import requests
+                ...
+        """
+
+    def on_result_found(self, original_node, updated_node):
+        new_args = self.replace_arg(original_node, "verify", "True")
+        return self.update_arg_target(updated_node, new_args)

src/codemodder/codemods/upgrade_sslcontext_tls.py

@@ -58,7 +58,7 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Arg):
             pos_to_match
         ) and self.filter_by_path_includes_or_excludes(pos_to_match):
             line_number = pos_to_match.start.line
-            self.CHANGES_IN_FILE.append(
+            self.file_context.codemod_changes.append(
                 Change(str(line_number), self.CHANGE_DESCRIPTION).to_json()
             )
 

src/codemodder/codemods/url_sandbox.py

@@ -49,7 +49,9 @@ def transform_module_impl(self, tree: cst.Module) -> cst.Module:
         )
         tree.visit(find_requests_visitor)
         if find_requests_visitor.nodes_to_change:
-            UrlSandbox.CHANGES_IN_FILE.extend(find_requests_visitor.changes_in_file)
+            self.file_context.codemod_changes.extend(
+                find_requests_visitor.changes_in_file
+            )
             new_tree = tree.visit(ReplaceNodes(find_requests_visitor.nodes_to_change))
             DependencyManager().add(["security==1.0.1"])
             # if it finds any request.get(...), try to remove the imports

src/codemodder/file_context.py

@@ -20,3 +20,4 @@ def __post_init__(self):
             self.line_include = []
         if self.line_exclude is None:
             self.line_exclude = []
+        self.codemod_changes = []

tests/codemods/base_codemod_test.py

@@ -1,4 +1,4 @@
-# pylint: disable=no-member,not-callable
+# pylint: disable=no-member,not-callable,attribute-defined-outside-init
 import libcst as cst
 from libcst.codemod import CodemodContext
 from pathlib import Path
@@ -12,20 +12,23 @@
 class BaseCodemodTest:
     codemod: ClassVar = NotImplemented
 
+    def setup_method(self):
+        self.file_context = None
+
     def run_and_assert(self, tmpdir, input_code, expected):
         tmp_file_path = tmpdir / "code.py"
         self.run_and_assert_filepath(tmpdir, tmp_file_path, input_code, expected)
 
     def run_and_assert_filepath(self, _, file_path, input_code, expected):
         input_tree = cst.parse_module(input_code)
-        file_context = FileContext(
+        self.file_context = FileContext(
             file_path,
             False,
             [],
             [],
             [],
         )
-        command_instance = self.codemod(CodemodContext(), file_context)
+        command_instance = self.codemod(CodemodContext(), self.file_context)
         output_tree = command_instance.transform_module(input_tree)
 
         assert output_tree.code == expected
@@ -44,14 +47,14 @@ def run_and_assert_filepath(self, root, file_path, input_code, expected):
         input_tree = cst.parse_module(input_code)
         all_results = self.results_by_id_filepath(input_code, root, file_path)
         results = all_results[str(file_path)]
-        file_context = FileContext(
+        self.file_context = FileContext(
             file_path,
             False,
             [],
             [],
             results,
         )
-        command_instance = self.codemod(CodemodContext(), file_context)
+        command_instance = self.codemod(CodemodContext(), self.file_context)
         output_tree = command_instance.transform_module(input_tree)
 
         assert output_tree.code == expected

tests/codemods/test_order_imports.py

@@ -10,7 +10,7 @@ def test_no_change(self, tmpdir):
 from b import c
 """
         self.run_and_assert(tmpdir, before, before)
-        assert len(self.codemod.CHANGES_IN_FILE) == 0
+        assert len(self.file_context.codemod_changes) == 0
 
     def test_separate_from_imports_and_regular(self, tmpdir):
         before = r"""import y
@@ -21,7 +21,7 @@ def test_separate_from_imports_and_regular(self, tmpdir):
 import y
 from a import c"""
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_consolidate_from_imports(self, tmpdir):
         before = r"""from a import a1
@@ -30,7 +30,7 @@ def test_consolidate_from_imports(self, tmpdir):
 
         after = r"""from a import a1, a2, a3"""
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_order_blocks_separately(self, tmpdir):
         before = r"""import x
@@ -46,7 +46,7 @@ def test_order_blocks_separately(self, tmpdir):
 import y"""
 
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 2
+        assert len(self.file_context.codemod_changes) == 2
 
     def test_preserve_comments(self, tmpdir):
         before = r"""# do not move
@@ -69,7 +69,7 @@ def test_preserve_comments(self, tmpdir):
 """
 
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_handle_star_imports(self, tmpdir):
         before = r"""from a import x
@@ -82,7 +82,7 @@ def test_handle_star_imports(self, tmpdir):
 from a import b, x"""
 
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_handle_composite_and_relative_imports(self, tmpdir):
         before = r"""from . import a
@@ -93,7 +93,7 @@ def test_handle_composite_and_relative_imports(self, tmpdir):
 from . import a"""
 
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_natural_order(self, tmpdir):
         before = """from a import Object11
@@ -104,7 +104,7 @@ def test_natural_order(self, tmpdir):
 """
 
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_wont_remove_unused_future(self, tmpdir):
         before = """from __future__ import absolute_import
@@ -113,7 +113,7 @@ def test_wont_remove_unused_future(self, tmpdir):
         after = before
 
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 0
+        assert len(self.file_context.codemod_changes) == 0
 
     def test_organize_by_sections(self, tmpdir):
         before = """from codemodder.codemods.transformations.clean_imports import CleanImports
@@ -134,7 +134,7 @@ def test_organize_by_sections(self, tmpdir):
 """
 
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_will_ignore_non_top_level(self, tmpdir):
         before = """import global2
@@ -159,7 +159,7 @@ def f():
     import global3
 """
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1
 
     def test_it_can_change_behavior(self, tmpdir):
         # note that c will change from b to e due to the sort
@@ -175,4 +175,4 @@ def test_it_can_change_behavior(self, tmpdir):
 c()
 """
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 1
+        assert len(self.file_context.codemod_changes) == 1

tests/codemods/test_remove_unnecessary_f_str.py

@@ -17,7 +17,7 @@ def test_no_change(self, tmpdir):
 good = f"wow i don't have args but don't mess my braces {{ up }}"
 """
         self.run_and_assert(tmpdir, before, before)
-        assert len(self.codemod.CHANGES_IN_FILE) == 0
+        assert len(self.file_context.codemod_changes) == 0
 
     def test_change(self, tmpdir):
         before = r"""
@@ -31,4 +31,4 @@ def test_change(self, tmpdir):
 bad: str = r'bad\d+'
 """
         self.run_and_assert(tmpdir, before, after)
-        assert len(self.codemod.CHANGES_IN_FILE) == 3
+        assert len(self.file_context.codemod_changes) == 3