ignore certain partners

mediathekview · May 9, 2024 · fc4bf1f · fc4bf1f
1 parent aeb33aa
commit fc4bf1f
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 17 deletions.
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/ArdConstants.java b/src/main/java/de/mediathekview/mserver/crawler/ard/ArdConstants.java
@@ -36,11 +36,10 @@ public class ArdConstants {
         "swr",
         "wdr",
         "one",
+        "funk",
         "alpha",
         "tagesschau24",
-        "funk",
-        "phoenix",
-        "arte"
+        "phoenix"
       };
 
   public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();

diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializer.java
@@ -172,21 +172,16 @@ public List<ArdFilmDto> deserialize(
     final Sender sender = ArdConstants.PARTNER_TO_SENDER.get(partner.orElse(""));
     final Optional<ArdVideoInfoDto> videoInfo = parseVideos(itemObject, title.orElse(""));
 
-    if(title.orElse("").contains("- Die Antwort auf fast")) {
-
-      System.out.println("stop");
-    }
-
     if (title.isEmpty() || topic.isEmpty() || videoInfo.isEmpty()) {
       return films;
     }
 
     if (sender == null) {
-      LOG.error("Missing Partner {}", partner.orElse(""));
       if (partner.isEmpty()) {
         LOG.error("Missing Partner Element {}", jsonElement);
+      } else {
+        LOG.error("Ignore Partner {}", partner.get());
       }
-
       return films;
     }
 

diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTeasersDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTeasersDeserializer.java
@@ -21,9 +21,13 @@ abstract class ArdTeasersDeserializer {
   private static final String ATTRIBUTE_ID = "id";
   private static final String ATTRIBUTE_NUMBER_OF_CLIPS = "numberOfClips";
 
+  private static final String ELEMENT_PUBLICATION_SERVICE = "publicationService";
+  private static final String ATTRIBUTE_PARTNER = "partner";
+
   Set<ArdFilmInfoDto> parseTeasers(final JsonArray teasers) {
     return StreamSupport.stream(teasers.spliterator(), true)
         .map(JsonElement::getAsJsonObject)
+        .filter(this::isRelevant)
         .map(this::toFilmInfo)
         .filter(Objects::nonNull)
         .collect(Collectors.toSet());
@@ -55,4 +59,18 @@ private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips)
     final String url = String.format(ArdConstants.ITEM_URL, id);
     return new ArdFilmInfoDto(id, url, numberOfClips);
   }
+
+  private boolean isRelevant(final JsonObject teaserObject) {
+    if (teaserObject.has(ELEMENT_PUBLICATION_SERVICE)) {
+      final JsonObject publicationService =
+              teaserObject.get(ELEMENT_PUBLICATION_SERVICE).getAsJsonObject();
+      final Optional<String> attributeAsString =
+              JsonUtils.getAttributeAsString(publicationService, ATTRIBUTE_PARTNER);
+      if (attributeAsString.isPresent()) {
+        return ArdConstants.PARTNER_TO_SENDER.get(attributeAsString.get()) != null;
+      }
+    }
+
+    return true;
+  }
 }
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicsLetterDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicsLetterDeserializer.java
@@ -23,6 +23,8 @@ public class ArdTopicsLetterDeserializer implements JsonDeserializer<PaginationU
   private static final String ELEMENT_TOTAL_ELEMENTS = "totalElements";
   private static final String ELEMENT_PAGE_SIZE = "pageSize";
   private static final String ELEMENT_PAGINATION = "pagination";
+  private static final String ELEMENT_PUBLICATION_SERVICE = "publicationService";
+  private static final String ATTRIBUTE_PARTNER = "partner";
 
   private static final String ATTRIBUTE_ID = "id";
 
@@ -78,14 +80,29 @@ private Set<CrawlerUrlDTO> parseTeaser(final JsonObject teaserObject) {
     } else {
       id = JsonUtils.getAttributeAsString(teaserObject, ATTRIBUTE_ID);
     }
-
-    id.ifPresent(
-        nonNullId ->
-            results.add(
-                new CrawlerUrlDTO(
-                    String.format(
-                        ArdConstants.TOPIC_URL, nonNullId, ArdConstants.TOPIC_PAGE_SIZE))));
+    if (isRelevant(teaserObject)) {
+      id.ifPresent(
+          nonNullId ->
+              results.add(
+                  new CrawlerUrlDTO(
+                      String.format(
+                          ArdConstants.TOPIC_URL, nonNullId, ArdConstants.TOPIC_PAGE_SIZE))));
+    }
 
     return results;
   }
+
+  private boolean isRelevant(final JsonObject teaserObject) {
+    if (teaserObject.has(ELEMENT_PUBLICATION_SERVICE)) {
+      final JsonObject publicationService =
+              teaserObject.get(ELEMENT_PUBLICATION_SERVICE).getAsJsonObject();
+      final Optional<String> attributeAsString =
+              JsonUtils.getAttributeAsString(publicationService, ATTRIBUTE_PARTNER);
+      if (attributeAsString.isPresent()) {
+        return ArdConstants.PARTNER_TO_SENDER.get(attributeAsString.get()) != null;
+      }
+    }
+
+    return true;
+  }
 }