Upgrade translation module to use better endpoint

See ssut/py-googletrans#268 (comment)

Author: CoolCat467

.pre-commit-config.yaml

@@ -19,11 +19,11 @@ repos:
       - id: sort-simple-yaml
         files: .pre-commit-config.yaml
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.10.0
+    rev: 25.1.0
     hooks:
       - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.6
     hooks:
       - id: ruff
         types: [file]
@@ -33,17 +33,17 @@ repos:
     hooks:
       - id: badgie
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         additional_dependencies:
           - tomli
   - repo: https://github.com/crate-ci/typos
-    rev: dictgen-v0.3.1
+    rev: typos-dict-v0.12.4
     hooks:
       - id: typos
   - repo: https://github.com/woodruffw/zizmor-pre-commit
-    rev: v1.0.0
+    rev: v1.3.1
     hooks:
       - id: zizmor
   - repo: local

src/subtitle_translate/translate.py

@@ -25,12 +25,11 @@
 
 import json
 import random
-import urllib.request
 from functools import partial
 from typing import TYPE_CHECKING, Any, Final, TypeVar, cast
-from urllib.parse import urlencode
 
 import httpx
+import orjson
 import trio
 
 from subtitle_translate import agents
@@ -70,22 +69,22 @@ async def collect(
 ##    return 'http://clients5.google.com/translate_a/t?'+urlencode(query)
 
 
-def get_translation_url(
-    sentence: str,
-    to_language: str,
-    source_language: str = "auto",
-) -> str:
-    """Return the URL you should visit to get query translated to language to_language."""
-    query = {
-        "client": "gtx",
-        "dt": "t",
-        "sl": source_language,
-        "tl": to_language,
-        "q": sentence,
-    }
-    return "https://translate.googleapis.com/translate_a/single?" + urlencode(
-        query,
-    )
+##def get_translation_url(
+##    sentence: str,
+##    to_language: str,
+##    source_language: str = "auto",
+##) -> str:
+##    """Return the URL you should visit to get query translated to language to_language."""
+##    query = {
+##        "client": "gtx",
+##        "dt": "t",
+##        "sl": source_language,
+##        "tl": to_language,
+##        "q": sentence,
+##    }
+##    return "https://translate.googleapis.com/translate_a/single?" + urlencode(
+##        query,
+##    )
 
 
 def process_response(result: list[str] | list[list[Any]]) -> str:
@@ -99,7 +98,7 @@ def process_response(result: list[str] | list[list[Any]]) -> str:
             part = next_
         else:
             raise ValueError(
-                f"Unexpected type {type(part)!r}, expected list or str",
+                f"Unexpected type {type(part)!r}, expected list or str\n{part = }",
             )
     raise RuntimeError("Unreachable")
 
@@ -114,23 +113,65 @@ def is_url(text: str) -> bool:
     )
 
 
-def translate_sync(
-    sentence: str,
-    to_lang: str,
-    source_lang: str = "auto",
-) -> str:
-    """Return sentence translated from source_lang to to_lang."""
-    if is_url(sentence):
-        # skip URLs
-        return sentence
-
-    # Get URL from function, which uses urllib to generate proper query
-    url = get_translation_url(sentence, to_lang, source_lang)
-    if not url.startswith("http"):
-        raise ValueError("URL not http(s), is this intended?")
-    with urllib.request.urlopen(url, timeout=0.5) as file:  # noqa: S310
-        request_result = json.loads(file.read())
-    return process_response(request_result)
+##def translate_sync(
+##    sentence: str,
+##    to_lang: str,
+##    source_lang: str = "auto",
+##) -> str:
+##    """Return sentence translated from source_lang to to_lang."""
+##    if is_url(sentence):
+##        # skip URLs
+##        return sentence
+##
+##    # Get URL from function, which uses urllib to generate proper query
+##    url = get_translation_url(sentence, to_lang, source_lang)
+##    if not url.startswith("http"):
+##        raise ValueError("URL not http(s), is this intended?")
+##    with urllib.request.urlopen(url, timeout=0.5) as file:
+##        request_result = json.loads(file.read())
+##    return process_response(request_result)
+
+
+##async def get_translated_coroutine(
+##    client: httpx.AsyncClient,
+##    sentence: str,
+##    to_lang: str,
+##    source_lang: str = "auto",
+##) -> str:
+##    """Return the sentence translated, asynchronously."""
+##    global AGENT  # pylint: disable=global-statement
+##
+##    if is_url(sentence):
+##        # skip URLs
+##        return sentence
+##    # Make sure we have a timeout, so that in the event of network failures
+##    # or something code doesn't get stuck
+##    # Get URL from function, which uses urllib to generate proper query
+##    url = get_translation_url(sentence, to_lang, source_lang)
+##
+##    headers = {
+##        "User-Agent": "",
+##        "Accept": "*/*",
+##        "Accept-Language": "en-US,en-GB; q=0.5",
+##        "Accept-Encoding": "gzip, deflate",
+##        "Connection": "keep-alive",
+##    }
+##
+##    while True:
+##        AGENT = (AGENT + 1) % len(agents.USER_AGENTS)
+##        headers["User-Agent"] = agents.USER_AGENTS[AGENT]
+##
+##        try:
+##            # Go to that URL and get our translated response
+##            response = await client.get(url, headers=headers)
+##            # Wait for our response and make it json so we can look at
+##            # it like a dictionary
+##            return process_response(response.json())
+##        except httpx.ConnectTimeout:
+##            pass
+##        except json.decoder.JSONDecodeError:
+##            print(f"{type(response) = }\n{response = }")
+##            raise
 
 
 async def get_translated_coroutine(
@@ -148,31 +189,33 @@ async def get_translated_coroutine(
     # Make sure we have a timeout, so that in the event of network failures
     # or something code doesn't get stuck
     # Get URL from function, which uses urllib to generate proper query
-    url = get_translation_url(sentence, to_lang, source_lang)
+    url = "https://translate-pa.googleapis.com/v1/translateHtml"
 
     headers = {
-        "User-Agent": "",
         "Accept": "*/*",
-        "Accept-Language": "en-US,en-GB; q=0.5",
-        "Accept-Encoding": "gzip, deflate",
-        "Connection": "keep-alive",
+        "X-Goog-API-Key": "AIzaSyATBXajvzQLTDHEQbcpq0Ihe0vWDHmO520",
+        "Content-Type": "application/json+protobuf",
     }
 
-    while True:
-        AGENT = (AGENT + 1) % len(agents.USER_AGENTS)
-        headers["User-Agent"] = agents.USER_AGENTS[AGENT]
+    sentence = sentence.replace("\n", "<br>")
+
+    raw_content = [[[sentence], source_lang, to_lang], "wt_lib"]
+    content = orjson.dumps(raw_content)
 
+    while True:
         try:
             # Go to that URL and get our translated response
-            response = await client.get(url, headers=headers)
+            response = await client.post(url, content=content, headers=headers)
             # Wait for our response and make it json so we can look at
             # it like a dictionary
-            return process_response(response.json())
+            return process_response(response.json()).replace("<br>", "\n")
         except httpx.ConnectTimeout:
             pass
         except json.decoder.JSONDecodeError:
             print(f"{type(response) = }\n{response = }")
             raise
+        AGENT = (AGENT + 1) % len(agents.USER_AGENTS)
+        headers["User-Agent"] = agents.USER_AGENTS[AGENT]
 
 
 async def translate_async(