From 636c4cd6b2b24bac733d232b1085402cb0479777 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sat, 23 Jan 2021 18:17:35 +0100 Subject: [PATCH] Support SoundCloud HLS by using a workaround This commit tries to support SoundCloud HLS streams by parsing M3U manifests, get the last segment URL (in order to get track length) and request a segment URL equals to track's duration so it's a single URL. Warning: This code doesn't work for now and throws a StreamExtractException --- .../extractors/SoundcloudStreamExtractor.java | 91 +++++++++++++++---- 1 file changed, 71 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 62e79cb2d1..4d6c594550 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -11,9 +11,9 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; @@ -33,11 +33,14 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudStreamExtractor extends StreamExtractor { @@ -179,7 +182,7 @@ public String getHlsUrl() { @Override public List getAudioStreams() throws IOException, ExtractionException { - List audioStreams = new ArrayList<>(); + final List audioStreams = new ArrayList<>(); final Downloader dl = NewPipe.getDownloader(); // Streams can be streamable and downloadable - or explicitly not. @@ -190,54 +193,102 @@ public List getAudioStreams() throws IOException, ExtractionExcepti try { final JsonArray transcodings = track.getObject("media").getArray("transcodings"); - // get information about what stream formats are available - for (Object transcoding : transcodings) { - + // Get information about what stream formats are available + for (final Object transcoding : transcodings) { final JsonObject t = (JsonObject) transcoding; String url = t.getString("url"); + String mediaUrl = null; + final MediaFormat mediaFormat; + final int bitrate; if (!isNullOrEmpty(url)) { + if (t.getString("preset").contains("mp3")) { + mediaFormat = MediaFormat.MP3; + bitrate = 128; + } else if (t.getString("preset").contains("opus")) { + mediaFormat = MediaFormat.OPUS; + bitrate = 64; + } else { + continue; + } - // We can only play the mp3 format, but not handle m3u playlists / streams. - // what about Opus? - if (t.getString("preset").contains("mp3") - && t.getObject("format").getString("protocol").equals("progressive")) { + // TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream) + + if (t.getObject("format").getString("protocol").equals("progressive")) { // This url points to the endpoint which generates a unique and short living url to the stream. - // TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream) url += "?client_id=" + SoundcloudParsingHelper.clientId(); final String res = dl.get(url).responseBody(); try { JsonObject mp3UrlObject = JsonParser.object().from(res); // Links in this file are also only valid for a short period. - audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), - MediaFormat.MP3, 128)); - } catch (JsonParserException e) { + mediaUrl = mp3UrlObject.getString("url"); + } catch (final JsonParserException e) { throw new ParsingException("Could not parse streamable url", e); } + } else if (t.getObject("format").getString("protocol").equals("hls")) { + // This url points to the endpoint which generates a unique and short living url to the stream. + url += "?client_id=" + SoundcloudParsingHelper.clientId(); + final String res = dl.get(url).responseBody(); + + try { + final JsonObject mp3HlsUrlObject = JsonParser.object().from(res); + // Links in this file are also only valid for a short period. + try { + mediaUrl = getUrlFromHlsManifest(mp3HlsUrlObject.getString("url")); + } catch (final ParsingException e) { + continue; + } + } catch (final JsonParserException e) { + throw new ParsingException("Could not parse streamable url", e); + } + } else { + continue; } + + audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate)); } } - } catch (NullPointerException e) { + } catch (final NullPointerException e) { throw new ExtractionException("Could not get SoundCloud's track audio url", e); } - if (audioStreams.isEmpty()) { - throw new ContentNotSupportedException("HLS audio streams are not yet supported"); - } - return audioStreams; } private static String urlEncode(String value) { try { return URLEncoder.encode(value, "UTF-8"); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { throw new IllegalStateException(e); } } + private String getUrlFromHlsManifest(final String hlsManifestUrl) throws ParsingException { + final Downloader dl = NewPipe.getDownloader(); + final String hlsManifestResponse; + + try { + hlsManifestResponse = dl.get(hlsManifestUrl).responseBody(); + } catch (final IOException | ReCaptchaException e) { + throw new ParsingException("Could not get SoundCloud HLS manifest"); + } + + final List hlsRangesList = new ArrayList(); + final Matcher regex = Pattern.compile("^(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]") + .matcher(hlsManifestResponse); + + while (regex.find()) { + hlsRangesList.add(hlsManifestResponse.substring(regex.start(0), regex.end(0))); + } + + final String hlsLastRangeUrl = hlsRangesList.get(hlsRangesList.size() - 1); + final String[] hlsLastRangeUrlArray = hlsLastRangeUrl.split("/"); + + return HTTPS + hlsLastRangeUrlArray[0] + "/media/0/" + hlsLastRangeUrlArray[3] + "/" + hlsLastRangeUrlArray[4]; + } + @Override public List getVideoStreams() { return Collections.emptyList(); @@ -334,4 +385,4 @@ public List getStreamSegments() { public List getMetaInfo() { return Collections.emptyList(); } -} +} \ No newline at end of file