fixup! Complete pip-tools setup (aio-libs#5486)

tools/deps.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
 import argparse
+import collections
+import hashlib
+import os
 import pathlib
 import platform
 import subprocess
@@ -13,11 +16,7 @@
 
 
 def txt_file_name(in_file_name):
-    txt_name = "{platform}-{py_version}-{in_file_name}.txt".format(
-        platform=PLATFORM_NAME,
-        py_version="{}.{}".format(sys.version_info.major, sys.version_info.minor),
-        in_file_name=in_file_name,
-    )
+    txt_name = f"{PLATFORM_NAME}-{sys.version_info.major}.{sys.version_info.minor}-{in_file_name}.txt"
     return PIN_FOLDER / txt_name
 
 
@@ -34,8 +33,8 @@ def compile_dependencies():
                 "--no-annotate",
                 "--quiet",
                 "--output-file",
-                output_file,
-                in_file,
+                str(output_file),
+                str(in_file),
             )
         )
 
@@ -49,17 +48,37 @@ def install_dependencies(in_name):
             "install",
             "--require-hashes",
             "-r",
-            txt_file,
+            str(txt_file),
         )
     )
 
 
+def replace_duplicates_with_symlinks():
+    hash_to_file = collections.defaultdict(list)
+
+    for txt_file_name in PIN_FOLDER.glob("*.txt"):
+        if not txt_file_name.is_symlink():
+            with open(txt_file_name, "rb") as f:
+                digest = hashlib.sha1(f.read()).hexdigest()
+                hash_to_file[digest].append(txt_file_name)
+
+    for digest, files in hash_to_file.items():
+        if len(files) > 1:
+            base_file, *other_files = files
+            for other_file in other_files:
+                os.unlink(other_file)
+                os.symlink(base_file, other_file)
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(dest="cmd", help="sub-command help")
 
     install_parser = subparsers.add_parser("install", help="install dependencies")
     compile_parser = subparsers.add_parser("compile", help="compile dependencies")
+    remove_duplicates = subparsers.add_parser(
+        "rdup", help="remove duplicate .txt files"
+    )
 
     install_parser.add_argument("file", type=str, help="dep to install", default="dev")
 
@@ -68,3 +87,5 @@ def install_dependencies(in_name):
         install_dependencies(args.file)
     elif args.cmd == "compile":
         compile_dependencies()
+    elif args.cmd == "rdup":
+        replace_duplicates_with_symlinks()