refactor: store hashed remote address on journals

tests/unit/cli/test_hashing.py

@@ -0,0 +1,116 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+
+import pretend
+
+from warehouse import db
+from warehouse.cli import hashing
+
+from ...common.db.packaging import JournalEntry, JournalEntryFactory
+
+
+def remote_addr_salty_hash(remote_addr, salt):
+    return hashlib.sha256(f"{remote_addr}{salt}".encode()).hexdigest()
+
+
+class TestHashingJournalEntry:
+    def test_no_records_to_hash(self, cli, db_request, monkeypatch):
+        engine = pretend.stub()
+        config = pretend.stub(registry={"sqlalchemy.engine": engine})
+        session_cls = pretend.call_recorder(lambda bind: db_request.db)
+        monkeypatch.setattr(db, "Session", session_cls)
+
+        assert db_request.db.query(JournalEntry).count() == 0
+
+        args = ["--salt", "test"]
+
+        result = cli.invoke(hashing.journal_entry, args, obj=config)
+
+        assert result.exit_code == 0
+        assert result.output.strip() == "No rows to hash. Done!"
+
+    def tests_hashes_records(self, cli, db_request, remote_addr, monkeypatch):
+        engine = pretend.stub()
+        config = pretend.stub(registry={"sqlalchemy.engine": engine})
+        session_cls = pretend.call_recorder(lambda bind: db_request.db)
+        monkeypatch.setattr(db, "Session", session_cls)
+
+        # create some JournalEntry records with unhashed ip addresses
+        JournalEntryFactory.create_batch(3, submitted_from=remote_addr)
+        assert db_request.db.query(JournalEntry).count() == 3
+
+        salt = "NaCl"
+        salted_hash = remote_addr_salty_hash(remote_addr, salt)
+
+        args = [
+            "--salt",
+            salt,
+            "--batch-size",
+            "2",
+        ]
+
+        result = cli.invoke(hashing.journal_entry, args, obj=config)
+
+        assert result.exit_code == 0
+        assert result.output.strip() == "Hashing 2 rows...\nHashed 2 rows"
+        # check that two of the ip addresses have been hashed
+        assert (
+            db_request.db.query(JournalEntry)
+            .filter_by(submitted_from=remote_addr)
+            .one()
+        )
+        assert (
+            db_request.db.query(JournalEntry)
+            .filter_by(submitted_from=salted_hash)
+            .count()
+            == 2
+        )
+
+    def test_continue_until_done(self, cli, db_request, remote_addr, monkeypatch):
+        engine = pretend.stub()
+        config = pretend.stub(registry={"sqlalchemy.engine": engine})
+        session_cls = pretend.call_recorder(lambda bind: db_request.db)
+        monkeypatch.setattr(db, "Session", session_cls)
+
+        # create some JournalEntry records with unhashed ip addresses
+        JournalEntryFactory.create_batch(3, submitted_from=remote_addr)
+
+        salt = "NaCl"
+        salted_hash = remote_addr_salty_hash(remote_addr, salt)
+
+        args = [
+            "--salt",
+            salt,
+            "--batch-size",
+            "1",
+            "--sleep-time",
+            "0",
+            "--continue-until-done",
+        ]
+
+        result = cli.invoke(hashing.journal_entry, args, obj=config)
+
+        assert result.exit_code == 0
+        # check that all the ip addresses have been hashed
+        assert (
+            db_request.db.query(JournalEntry)
+            .filter_by(submitted_from=salted_hash)
+            .count()
+            == 3
+        )
+        assert (
+            db_request.db.query(JournalEntry)
+            .filter_by(submitted_from=remote_addr)
+            .count()
+            == 0
+        )

tests/unit/forklift/test_legacy.py

@@ -1481,7 +1481,7 @@ def storage_service_store(path, file_path, *, meta):
                 release.version,
                 f"add source file {filename}",
                 user,
-                db_request.remote_addr,
+                db_request.remote_addr_hashed,
             )
         ]
 
@@ -2122,21 +2122,27 @@ def test_upload_succeeds_custom_project_size_limit(
             (j.name, j.version, j.action, j.submitted_by, j.submitted_from)
             for j in journals
         ] == [
-            ("example", None, "create", user, db_request.remote_addr),
+            ("example", None, "create", user, db_request.ip_address.hashed_ip_address),
             (
                 "example",
                 None,
                 f"add Owner {user.username}",
                 user,
-                db_request.remote_addr,
+                db_request.ip_address.hashed_ip_address,
+            ),
+            (
+                "example",
+                "1.0",
+                "new release",
+                user,
+                db_request.ip_address.hashed_ip_address,
             ),
-            ("example", "1.0", "new release", user, db_request.remote_addr),
             (
                 "example",
                 "1.0",
                 "add source file example-1.0.tar.gz",
                 user,
-                db_request.remote_addr,
+                db_request.ip_address.hashed_ip_address,
             ),
         ]
 
@@ -2777,7 +2783,7 @@ def storage_service_store(path, file_path, *, meta):
                 release.version,
                 f"add cp34 file {filename}",
                 user,
-                db_request.remote_addr,
+                db_request.remote_addr_hashed,
             )
         ]
 
@@ -2911,7 +2917,7 @@ def storage_service_store(path, file_path, *, meta):
                 release.version,
                 f"add cp34 file {filename}",
                 user,
-                db_request.remote_addr,
+                db_request.remote_addr_hashed,
             )
         ]
 
@@ -3176,14 +3182,14 @@ def test_upload_succeeds_creates_release(
                 release.version,
                 "new release",
                 user,
-                db_request.remote_addr,
+                db_request.remote_addr_hashed,
             ),
             (
                 release.project.name,
                 release.version,
                 f"add source file {filename}",
                 user,
-                db_request.remote_addr,
+                db_request.remote_addr_hashed,
             ),
         ]
 
@@ -3468,21 +3474,27 @@ def test_upload_succeeds_creates_project(
             (j.name, j.version, j.action, j.submitted_by, j.submitted_from)
             for j in journals
         ] == [
-            ("example", None, "create", user, db_request.remote_addr),
+            ("example", None, "create", user, db_request.ip_address.hashed_ip_address),
             (
                 "example",
                 None,
                 f"add Owner {user.username}",
                 user,
-                db_request.remote_addr,
+                db_request.ip_address.hashed_ip_address,
+            ),
+            (
+                "example",
+                "1.0",
+                "new release",
+                user,
+                db_request.ip_address.hashed_ip_address,
             ),
-            ("example", "1.0", "new release", user, db_request.remote_addr),
             (
                 "example",
                 "1.0",
                 "add source file example-1.0.tar.gz",
                 user,
-                db_request.remote_addr,
+                db_request.ip_address.hashed_ip_address,
             ),
         ]
 

tests/unit/manage/test_views.py

@@ -3975,7 +3975,7 @@ def test_yank_project_release(self, monkeypatch, db_request):
         assert entry.action == "yank release"
         assert entry.version == release.version
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
         assert db_request.session.flash.calls == [
             pretend.call(f"Yanked release {release.version!r}", queue="success")
         ]
@@ -4129,7 +4129,7 @@ def test_unyank_project_release(self, monkeypatch, db_request):
         assert entry.action == "unyank release"
         assert entry.version == release.version
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
 
         assert db_request.session.flash.calls == [
             pretend.call(f"Un-yanked release {release.version!r}", queue="success")
@@ -4287,7 +4287,7 @@ def test_delete_project_release(self, monkeypatch, db_request):
         assert entry.action == "remove release"
         assert entry.version == release.version
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
 
         assert db_request.session.flash.calls == [
             pretend.call(f"Deleted release {release.version!r}", queue="success")
@@ -4474,7 +4474,7 @@ def test_delete_project_release_file(self, monkeypatch, db_request):
                 version=release.version,
                 action=f"remove file {release_file.filename}",
                 submitted_by=user,
-                submitted_from=db_request.remote_addr,
+                submitted_from=db_request.ip_address.hashed_ip_address,
             )
             .one()
         )
@@ -5460,7 +5460,7 @@ def test_change_role(self, db_request, monkeypatch):
         assert entry.name == project.name
         assert entry.action == "change Owner testuser to Maintainer"
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
 
     def test_change_role_invalid_role_name(self, pyramid_request):
         project = pretend.stub(name="foobar")
@@ -5578,7 +5578,7 @@ def test_delete_role(self, db_request, monkeypatch):
         assert entry.name == project.name
         assert entry.action == "remove Owner testuser"
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
 
     def test_delete_missing_role(self, db_request):
         project = ProjectFactory.create(name="foobar")
@@ -5674,7 +5674,7 @@ def test_delete_not_sole_owner_role(self, db_request, monkeypatch):
         assert entry.name == project.name
         assert entry.action == "remove Owner testuser"
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
 
     def test_delete_non_owner_role(self, db_request):
         project = ProjectFactory.create(name="foobar")

tests/unit/manage/views/test_teams.py

@@ -838,7 +838,7 @@ def test_change_role(
         assert entry.name == organization_project.name
         assert entry.action == f"change Owner {organization_team.name} to Maintainer"
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
 
     def test_change_role_invalid_role_name(self, pyramid_request, organization_project):
         pyramid_request.method = "POST"
@@ -1023,7 +1023,7 @@ def test_delete_role(
         assert entry.name == organization_project.name
         assert entry.action == f"remove Owner {organization_team.name}"
         assert entry.submitted_by == db_request.user
-        assert entry.submitted_from == db_request.remote_addr
+        assert entry.submitted_from == db_request.ip_address.hashed_ip_address
 
     def test_delete_missing_role(self, db_request, organization_project):
         missing_role_id = str(uuid.uuid4())

tests/unit/utils/test_project.py

@@ -140,7 +140,7 @@ def test_remove_project(db_request, flash):
     )
     assert journal_entry.action == "remove project"
     assert journal_entry.submitted_by == db_request.user
-    assert journal_entry.submitted_from == db_request.remote_addr
+    assert journal_entry.submitted_from == db_request.ip_address.hashed_ip_address
 
 
 @pytest.mark.parametrize("flash", [True, False])
@@ -164,7 +164,7 @@ def test_destroy_docs(db_request, flash):
     )
     assert journal_entry.action == "docdestroy"
     assert journal_entry.submitted_by == db_request.user
-    assert journal_entry.submitted_from == db_request.remote_addr
+    assert journal_entry.submitted_from == db_request.ip_address.hashed_ip_address
 
     assert not (
         db_request.db.query(Project)

warehouse/accounts/views.py

@@ -1127,7 +1127,7 @@ def _error(message):
             name=project.name,
             action=f"accepted {desired_role} {user.username}",
             submitted_by=request.user,
-            submitted_from=request.remote_addr,
+            submitted_from=request.ip_address.hashed_ip_address,
         )
     )
     project.record_event(

warehouse/admin/views/projects.py

@@ -404,7 +404,7 @@ def add_role(project, request):
             name=project.name,
             action=f"add {role_name} {user.username}",
             submitted_by=request.user,
-            submitted_from=request.remote_addr,
+            submitted_from=request.ip_address.hashed_ip_address,
         )
     )
 
@@ -455,7 +455,7 @@ def delete_role(project, request):
             name=project.name,
             action=f"remove {role.role_name} {role.user.username}",
             submitted_by=request.user,
-            submitted_from=request.remote_addr,
+            submitted_from=request.ip_address.hashed_ip_address,
         )
     )
 

warehouse/admin/views/users.py

@@ -207,7 +207,7 @@ def _nuke_user(user, request):
                 name=project.name,
                 action="remove project",
                 submitted_by=request.user,
-                submitted_from=request.remote_addr,
+                submitted_from=request.ip_address.hashed_ip_address,
             )
         )
     projects.delete(synchronize_session=False)
@@ -239,7 +239,7 @@ def _nuke_user(user, request):
             name=f"user:{user.username}",
             action="nuke user",
             submitted_by=request.user,
-            submitted_from=request.remote_addr,
+            submitted_from=request.ip_address.hashed_ip_address,
         )
     )