Merge pull request #538 from B0pol/soundcloud-georestriction

Soundcloud: fix charts extraction when georestricted
TeamNewPipe · Feb 14, 2021 · d7486f2 · d7486f2
2 parents 432c68c + 8d89c82
commit d7486f2
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 9 deletions.
diff --git a/...c/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/...c/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
@@ -148,12 +148,12 @@ public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException
      */
     public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
         // Remove the tailing slash from URLs due to issues with the SoundCloud API
-        if (urlString.charAt(urlString.length() -1) == '/') urlString = urlString.substring(0, urlString.length()-1);
+        if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
 
         URL url;
         try {
             url = Utils.stringToURL(urlString);
-        } catch (MalformedURLException e){
+        } catch (MalformedURLException e) {
             throw new IllegalArgumentException("The given URL is not valid");
         }
 
@@ -240,10 +240,14 @@ public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsColl
      * @return the next streams url, empty if don't have
      */
     public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
-        String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
+        final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
+        if (response.responseCode() >= 400) {
+            throw new IOException("Could not get streams from API, HTTP " + response.responseCode());
+        }
+
         JsonObject responseObject;
         try {
-            responseObject = JsonParser.object().from(response);
+            responseObject = JsonParser.object().from(response.responseBody());
         } catch (JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }

diff --git a/...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java b/...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
@@ -6,13 +6,13 @@
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
-import java.io.IOException;
-
 import javax.annotation.Nonnull;
+import java.io.IOException;
 
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -61,10 +61,20 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
             apiUrl += "&kind=trending";
         }
 
-        final String contentCountry = SoundCloud.getContentCountry().getCountryCode();
-        apiUrl += "&region=soundcloud:regions:" + contentCountry;
+        final ContentCountry contentCountry = SoundCloud.getContentCountry();
+        String apiUrlWithRegion = null;
+        if (getService().getSupportedCountries().contains(contentCountry)) {
+            apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
+        }
 
-        final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
+        String nextPageUrl;
+        try {
+            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
+        } catch (IOException e) {
+            // Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
+            // we retry without the specified region.
+            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
+        }
 
         return new InfoItemsPage<>(collector, new Page(nextPageUrl));
     }