Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix dw resolutions #863

Merged
merged 1 commit into from
Dec 18, 2022
Merged

fix dw resolutions #863

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 15 additions & 24 deletions src/main/java/de/mediathekview/mserver/crawler/dw/DwVideoDto.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,35 +1,26 @@
package de.mediathekview.mserver.crawler.dw;

import java.net.URL;

public class DwVideoDto {

private int bitrate = 0;
private String quality = null;
private String format = null;
private URL url = null;

public DwVideoDto(int bitrate, String quality, String format, URL url) {
this.bitrate = bitrate;
this.quality = quality;
private final int bitRate;
private final String url;
private final int width;

public DwVideoDto(String url, int width, int bitRate) {
this.url = url;
this.format = format;
}

public int getBitrate() {
return bitrate;
this.width = width;
this.bitRate = bitRate;
}
public String getQuality() {
return quality;

public int getBitRate() {
return bitRate;
}
public URL getUrl() {

public String getUrl() {
return url;
}
public String getFormat() {
return format;

public int getWidth() {
return width;
}

}
207 changes: 128 additions & 79 deletions src/main/java/de/mediathekview/mserver/crawler/dw/parser/DwFilmDetailDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,28 @@
package de.mediathekview.mserver.crawler.dw.parser;

import com.google.gson.JsonArray;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;

import com.google.gson.*;
import de.mediathekview.mlib.daten.Film;
import de.mediathekview.mlib.daten.FilmUrl;
import de.mediathekview.mlib.daten.Resolution;
import de.mediathekview.mlib.daten.Sender;
import de.mediathekview.mserver.base.utils.JsonUtils;
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
import de.mediathekview.mserver.crawler.dw.DwVideoDto;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.lang.reflect.Type;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

public class DwFilmDetailDeserializer
implements JsonDeserializer<Optional<Film>> {
public class DwFilmDetailDeserializer implements JsonDeserializer<Optional<Film>> {
private static final Logger LOG = LogManager.getLogger(DwFilmDetailDeserializer.class);

private static final String ELEMENT_ID = "id";
Expand All @@ -41,15 +32,18 @@ public class DwFilmDetailDeserializer
private static final String ELEMENT_CATEGORY = "categoryName";
private static final String ELEMENT_LINK = "permaLink";
private static final String ELEMENT_DATETIME = "displayDate";

private static final String ELEMENT_MAINCONTENT = "mainContent";
private static final String ELEMENT_MAINCONTENT_LINK = "url";
private static final String ELEMENT_MAINCONTENT_DURATION = "duration";
private static final String ELEMENT_MAINCONTENT_SOURCES = "sources";
private static final String ELEMENT_MAINCONTENT_SOURCES_QUALITY = "quality";
private static final String ELEMENT_MAINCONTENT_SOURCES_URL = "url";
private static final String ELEMENT_MAINCONTENT_SOURCES_FORMAT = "format";
private static final String ELEMENT_MAINCONTENT_SOURCES_BITRATE = "bitrate";
private static final Pattern PATTERN_VIDEO_WITH_RESOLUTION =
Pattern.compile(".+_(\\d+)x(\\d+)\\.mp4");
private static final Comparator<? super DwVideoDto> DW_VIDEO_COMPARATOR =
Comparator.comparing(DwVideoDto::getWidth).thenComparing(DwVideoDto::getBitRate);

private final Sender sender;
private final AbstractCrawler crawler;
Expand All @@ -59,7 +53,12 @@ public DwFilmDetailDeserializer(final AbstractCrawler aCrawler) {
crawler = aCrawler;
}

protected boolean isValidVideo(final JsonObject jsonObject, Optional<String> videoId, Optional<String> type, Optional<String> title, Optional<String> topic) {
protected boolean isValidVideo(
final JsonObject jsonObject,
Optional<String> videoId,
Optional<String> type,
Optional<String> title,
Optional<String> topic) {
if (videoId.isEmpty()) {
LOG.error("Could not find mandatory element videoId");
return false;
Expand All @@ -77,16 +76,21 @@ protected boolean isValidVideo(final JsonObject jsonObject, Optional<String> vid
} else if (!jsonObject.has(ELEMENT_MAINCONTENT)) {
LOG.error("Could not find maincontent for videoId {}", videoId.get());
return false;
} else if (!jsonObject.get(ELEMENT_MAINCONTENT).getAsJsonObject().has(ELEMENT_MAINCONTENT_SOURCES)) {
} else if (!jsonObject
.get(ELEMENT_MAINCONTENT)
.getAsJsonObject()
.has(ELEMENT_MAINCONTENT_SOURCES)) {
LOG.error("Could not find sources for videoId {}", videoId.get());
return false;
} else if (JsonUtils.getAttributeAsString(jsonObject.get(ELEMENT_MAINCONTENT).getAsJsonObject(), ELEMENT_MAINCONTENT_LINK).isEmpty()) {
} else if (JsonUtils.getAttributeAsString(
jsonObject.get(ELEMENT_MAINCONTENT).getAsJsonObject(), ELEMENT_MAINCONTENT_LINK)
.isEmpty()) {
LOG.error("Could not find thisPageUrl for videoId {}", videoId.get());
return false;
}
return true;
}

@Override
public Optional<Film> deserialize(
JsonElement aJsonElement, Type aType, JsonDeserializationContext aContext) {
Expand All @@ -103,22 +107,30 @@ public Optional<Film> deserialize(
}

final JsonObject jsonObjectMainContent = jsonObject.get(ELEMENT_MAINCONTENT).getAsJsonObject();
final Optional<String> thisPageUrl = JsonUtils.getAttributeAsString(jsonObjectMainContent, ELEMENT_MAINCONTENT_LINK);
final Optional<String> thisPageUrl =
JsonUtils.getAttributeAsString(jsonObjectMainContent, ELEMENT_MAINCONTENT_LINK);

final Film film =
new Film(UUID.randomUUID(), sender, title.get(), topic.get(), getAiredDate(thisPageUrl.get(),jsonObject), getDuration(thisPageUrl.get(),jsonObjectMainContent));
new Film(
UUID.randomUUID(),
sender,
title.get(),
topic.get(),
getAiredDate(thisPageUrl.get(), jsonObject),
getDuration(thisPageUrl.get(), jsonObjectMainContent));
//
final Optional<String> description = JsonUtils.getAttributeAsString(jsonObject, ELEMENT_TEASER);
description.ifPresent(film::setBeschreibung);
//
getWebsite(thisPageUrl.get(), jsonObject).ifPresent(film::setWebsite);
//
final JsonArray jsonObjectMainContentSources = jsonObjectMainContent.get(ELEMENT_MAINCONTENT_SOURCES).getAsJsonArray();
getVideos(title.get(),jsonObjectMainContentSources).ifPresent(film::addAllUrls);
final JsonArray jsonObjectMainContentSources =
jsonObjectMainContent.get(ELEMENT_MAINCONTENT_SOURCES).getAsJsonArray();
getVideos(title.get(), jsonObjectMainContentSources).ifPresent(film::addAllUrls);
//
return Optional.of(film);
}

private Optional<URL> getWebsite(final String videoid, final JsonObject jsonObject) {
Optional<URL> websiteUrl = Optional.empty();
final Optional<String> websiteString = JsonUtils.getAttributeAsString(jsonObject, ELEMENT_LINK);
Expand All @@ -133,33 +145,36 @@ private Optional<URL> getWebsite(final String videoid, final JsonObject jsonObje
}
return websiteUrl;
}

private LocalDateTime getAiredDate(final String videoid, final JsonObject jsonObject) {
final DateTimeFormatter dateFormatter =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX");
LocalDateTime airedDatetime = LocalDateTime.now();
final Optional<String> displayDate = JsonUtils.getAttributeAsString(jsonObject, ELEMENT_DATETIME);
final Optional<String> displayDate =
JsonUtils.getAttributeAsString(jsonObject, ELEMENT_DATETIME);
if (displayDate.isPresent()) {
try {
airedDatetime = LocalDateTime.parse(displayDate.get(), dateFormatter);
} catch (Exception e) {
LOG.error("error parsing getAiredDate value '{}' for video {}", displayDate.get(), videoid);
}
} else {
LOG.error("no airedDate found for video {}", videoid);
LOG.error("no airedDate found for video {}", videoid);
}
return airedDatetime;
}

private Duration getDuration(final String videoid, final JsonObject jsonObject) {
Duration duration = Duration.ofSeconds(0);
final Optional<String> durationString = JsonUtils.getAttributeAsString(jsonObject, ELEMENT_MAINCONTENT_DURATION);
final Optional<String> durationString =
JsonUtils.getAttributeAsString(jsonObject, ELEMENT_MAINCONTENT_DURATION);
if (durationString.isPresent()) {
try {
Optional<String> durationInSeconds = JsonUtils.getAttributeAsString(jsonObject, ELEMENT_MAINCONTENT_DURATION);
if (durationInSeconds.isPresent()) {
duration = Duration.ofSeconds(Integer.parseInt(durationInSeconds.get()));
}
Optional<String> durationInSeconds =
JsonUtils.getAttributeAsString(jsonObject, ELEMENT_MAINCONTENT_DURATION);
if (durationInSeconds.isPresent()) {
duration = Duration.ofSeconds(Integer.parseInt(durationInSeconds.get()));
}
} catch (Exception e) {
LOG.error("error getDuration for video {} on value '{}'", videoid, durationString.get());
}
Expand All @@ -169,7 +184,8 @@ private Duration getDuration(final String videoid, final JsonObject jsonObject)
return duration;
}

private Optional<Map<Resolution, FilmUrl>> getVideos(final String videoid, final JsonArray videos) {
private Optional<Map<Resolution, FilmUrl>> getVideos(
final String videoid, final JsonArray videos) {

if (videos == null) {
return Optional.empty();
Expand All @@ -179,59 +195,92 @@ private Optional<Map<Resolution, FilmUrl>> getVideos(final String videoid, final
final ArrayList<DwVideoDto> videoListeRaw = new ArrayList<>();

videos.forEach(
(JsonElement currentElement) -> {
if (currentElement.isJsonObject()) {
final JsonObject currentElementObject = currentElement.getAsJsonObject();
final Optional<String> quality = JsonUtils.getAttributeAsString(currentElementObject, ELEMENT_MAINCONTENT_SOURCES_QUALITY);
final Optional<String> bitrate = JsonUtils.getAttributeAsString(currentElementObject, ELEMENT_MAINCONTENT_SOURCES_BITRATE);
final Optional<String> format = JsonUtils.getAttributeAsString(currentElementObject, ELEMENT_MAINCONTENT_SOURCES_FORMAT);
final Optional<String> url = JsonUtils.getAttributeAsString(currentElementObject, ELEMENT_MAINCONTENT_SOURCES_URL);

if (url.isPresent() && quality.isPresent() && format.isPresent() && bitrate.isPresent()) {
try {
(JsonElement currentElement) -> {
if (currentElement.isJsonObject()) {
final JsonObject currentElementObject = currentElement.getAsJsonObject();
final Optional<String> bitrateValue =
JsonUtils.getAttributeAsString(
currentElementObject, ELEMENT_MAINCONTENT_SOURCES_BITRATE);
final Optional<String> format =
JsonUtils.getAttributeAsString(
currentElementObject, ELEMENT_MAINCONTENT_SOURCES_FORMAT);
final Optional<String> url =
JsonUtils.getAttributeAsString(
currentElementObject, ELEMENT_MAINCONTENT_SOURCES_URL);

if (url.isPresent() && format.isPresent()) {
if (!format.get().equalsIgnoreCase("hls")) {
videoListeRaw.add(new DwVideoDto(Integer.parseInt(bitrate.get()), quality.get(), format.get(), new URL(url.get())));
final int bitrate = tryParseBitRate(bitrateValue);
final Optional<Integer> width = tryToDetermineWidthFromUrl(url.get());
videoListeRaw.add(new DwVideoDto(url.get(), width.orElse(0), bitrate));
}
} catch (final MalformedURLException e) {
// Nothing to be done here
LOG.error("Malformed video url for video: {}", videoid);
} else {
LOG.error("Mising video url element for video: {}", videoid);
}
} else {
LOG.error("Mising video url element for video: {}", videoid);
}
}
}
);
//
videoListeRaw.sort(Comparator.comparing(DwVideoDto::getBitrate));
for (int quality = 0; quality < videoListeRaw.size(); quality++) {
final FilmUrl filmUrl = new FilmUrl(videoListeRaw.get(quality).getUrl(), crawler.determineFileSizeInKB(videoListeRaw.get(quality).getUrl().toExternalForm()));
videoListe.put(getResolutionFromPosition(quality), filmUrl);
}
});
//
videoListeRaw.sort(DW_VIDEO_COMPARATOR);
videoListeRaw.forEach(
video -> {
try {
final Resolution resolution = determineResolution(video.getUrl(), video.getWidth());

final FilmUrl filmUrl =
new FilmUrl(video.getUrl(), crawler.determineFileSizeInKB(video.getUrl()));
videoListe.put(resolution, filmUrl);
} catch (final MalformedURLException e) {
// Nothing to be done here
LOG.error("Malformed video url for video: {}", videoid);
}
});

if (videoListe.size() > 0) {
return Optional.of(videoListe);
}
LOG.error("No video url for video: {}", videoid);
return Optional.empty();
}

private Resolution getResolutionFromPosition(int pos) {
switch (pos) {
case 0: {
return Resolution.VERY_SMALL;
}
case 1: {
return Resolution.SMALL;
}
case 2: {
return Resolution.NORMAL;

private int tryParseBitRate(Optional<String> bitRateValue) {
if (bitRateValue.isPresent()) {
try {
return Integer.parseInt(bitRateValue.get());
} catch (NumberFormatException e) {
LOG.debug(e);
}
}
return 0;
}

private Resolution determineResolution(String url, int width) {
if (width > 0) {
return Resolution.getResolutionFromWidth(width);
}
if (url.endsWith("avc.mp4") || url.endsWith("hd.mp4")) {
return Resolution.HD;
}
if (url.endsWith("sor.mp4")) {
return Resolution.SMALL;
}
if (url.endsWith("sd.mp4")) {
return Resolution.NORMAL;
}
LOG.debug("unknown url format: {}", url);
return Resolution.VERY_SMALL;
}

private Optional<Integer> tryToDetermineWidthFromUrl(String url) {

final Matcher matcher = PATTERN_VIDEO_WITH_RESOLUTION.matcher(url);
if (matcher.matches()) {
try {
final int width = Integer.parseInt(matcher.group(1));
return Optional.of(width);
} catch (NumberFormatException e) {
LOG.error(e);
}
case 3: {
return Resolution.HD;
}
default:
return Resolution.UHD;
}
}
return Optional.empty();
}
}
Loading