ard: increase deep search

ARTE: replace non reachable host in link
mediathekview · Feb 6, 2025 · 9953f5d · 9953f5d
2 parents 2746a8d + 8d93ad4
commit 9953f5d
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 5 deletions.
diff --git a/build.gradle b/build.gradle
@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.244'
+version = '3.1.245'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java
@@ -83,6 +83,8 @@ public class ArdTopicPageTask extends ArdTaskBase<ArdFilmInfoDto, CrawlerUrlDTO>
     TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3N0dXJtIGRlciBsaWViZQ");
     // in aller freundschaft -die jungen ärzte
     TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL21kci5kZS9zZW5kZXJlaWhlbi9zdGFmZmVsc2VyaWUtaW4tYWxsZXItZnJldW5kc2NoYWZ0LWRpZS1qdW5nZW4tYWVyenRl");
+    // tokyo vice
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3Rva3lvLXZpY2U");
     // um Himmels willen
     TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3dkci5kZS9VbSBIaW1tZWxzIFdpbGxlbiBPTkU");
     // Rentnercops

diff --git a/src/main/java/mServer/crawler/sender/arte/ArteListBaseDeserializer.java b/src/main/java/mServer/crawler/sender/arte/ArteListBaseDeserializer.java
@@ -16,24 +16,32 @@ public abstract class ArteListBaseDeserializer {
   private static final String JSON_ELEMENT_LINKS = "links";
   private static final String JSON_ELEMENT_NEXT = "next";
 
+  private static String buildUrl(String nextUrl) {
+    return UrlUtils.addDomainIfMissing(
+            nextUrl
+                    .replace("/api/emac/", "/api/rproxy/emac/")
+                    // fix non reachable host
+                    .replace("api-internal.infra-priv.arte.tv", "www.arte.tv")
+            , "https://www.arte.tv");
+  }
+
   protected Optional<String> parsePagination(JsonObject jsonObject) {
     if (jsonObject.has(JSON_ELEMENT_PAGINATION) && !jsonObject.get(JSON_ELEMENT_PAGINATION).isJsonNull()) {
       final JsonObject pagionationObject = jsonObject.get(JSON_ELEMENT_PAGINATION).getAsJsonObject();
-      if(pagionationObject.has(JSON_ELEMENT_LINKS)) {
+      if (pagionationObject.has(JSON_ELEMENT_LINKS)) {
         final JsonObject linksObject = pagionationObject.get(JSON_ELEMENT_LINKS).getAsJsonObject();
         final Optional<String> nextUrl = JsonUtils.getAttributeAsString(linksObject, JSON_ELEMENT_NEXT);
         if (nextUrl.isPresent()) {
-          return Optional.of(UrlUtils.addDomainIfMissing(nextUrl.get().replace("/api/emac/", "/api/rproxy/emac/"), "https://www.arte.tv"));
+          return Optional.of(buildUrl(nextUrl.get()));
         }
       }
     }
     return Optional.empty();
   }
 
-
   protected void extractProgramIdFromData(JsonObject jsonObectWithData, ArteCategoryFilmsDTO dto) {
     if (jsonObectWithData.has(JSON_ELEMENT_DATA)) {
-      for(JsonElement dataElement : jsonObectWithData.get(JSON_ELEMENT_DATA).getAsJsonArray()) {
+      for (JsonElement dataElement : jsonObectWithData.get(JSON_ELEMENT_DATA).getAsJsonArray()) {
         if (!dataElement.getAsJsonObject().get(JSON_ELEMENT_PROGRAMID).isJsonNull()) {
           Optional<String> programId = JsonUtils.getAttributeAsString(dataElement.getAsJsonObject(), JSON_ELEMENT_PROGRAMID);
           if (programId.isPresent()) {