Merge pull request #540 from B0pol/fix-tests

fix tests, fix links in YouTube description

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java

@@ -50,6 +50,9 @@ public String getUrl(String id, String baseUrl) throws ParsingException {
      * @return a {@link LinkHandler} complete with information
      */
     public LinkHandler fromUrl(final String url) throws ParsingException {
+        if (Utils.isNullOrEmpty(url)) {
+            throw new IllegalArgumentException("The url is null or empty");
+        }
         final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
         final String baseUrl = Utils.getBaseUrl(polishedUrl);
         return fromUrl(polishedUrl, baseUrl);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -429,10 +429,17 @@ public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaExcept
         return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
     }
 
+
+
     @Nullable
     public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
         if (navigationEndpoint.has("urlEndpoint")) {
             String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
+            if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
+                // remove https://www.youtube.com part to fall in the next if block
+                internUrl = internUrl.substring(23);
+            }
+
             if (internUrl.startsWith("/redirect?")) {
                 // q parameter can be the first parameter
                 internUrl = internUrl.substring(10);
@@ -450,6 +457,8 @@ public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint)
                 }
             } else if (internUrl.startsWith("http")) {
                 return internUrl;
+            } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") || internUrl.startsWith("/watch")) {
+                return "https://www.youtube.com" + internUrl;
             }
         } else if (navigationEndpoint.has("browseEndpoint")) {
             final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -224,7 +224,7 @@ public String getThumbnailUrl() throws ParsingException {
 
     @Nonnull
     @Override
-    public Description getDescription() {
+    public Description getDescription() throws ParsingException {
         assertPageFetched();
         // description with more info on links
         try {
@@ -234,8 +234,15 @@ public Description getDescription() {
             // age-restricted videos cause a ParsingException here
         }
 
+        String description = playerResponse.getObject("videoDetails").getString("shortDescription");
+        if (description == null) {
+            final JsonObject descriptionObject = playerResponse.getObject("microformat")
+                    .getObject("playerMicroformatRenderer").getObject("description");
+            description = getTextFromObject(descriptionObject);
+        }
+
         // raw non-html description
-        return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
+        return new Description(description, Description.PLAIN_TEXT);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -69,7 +69,15 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
     @Nonnull
     @Override
     public String getName() throws ParsingException {
-        String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
+        final JsonObject header = initialData.getObject("header");
+        JsonObject title = null;
+        if (header.has("feedTabbedHeaderRenderer")) {
+            title = header.getObject("feedTabbedHeaderRenderer").getObject("title");
+        } else if (header.has("c4TabbedHeaderRenderer")) {
+            title = header.getObject("c4TabbedHeaderRenderer").getObject("title");
+        }
+
+        String name = getTextFromObject(title);
         if (!isNullOrEmpty(name)) {
             return name;
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCRecentListExtractorTest.java

@@ -26,16 +26,13 @@ public static void setUpClass() throws Exception {
     }
 
     @Test
-    @Ignore("TODO fix")
     public void testStreamList() throws Exception {
         final List<StreamInfoItem> items = extractor.getInitialPage().getItems();
         assertEquals(100, items.size());
         for (final StreamInfoItem item: items) {
             assertFalse(isNullOrEmpty(item.getName()));
             assertTrue(item.getDuration() > 0);
-            assertTrue(isNullOrEmpty(item.getUploaderName())); // we do not get the uploader name
             assertTrue(item.getUploadDate().offsetDateTime().isBefore(OffsetDateTime.now()));
-            assertTrue(item.getUploadDate().offsetDateTime().isAfter(OffsetDateTime.now().minusYears(1)));
         }
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java

@@ -101,7 +101,6 @@ public void testFeedUrl() throws ParsingException {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testSubscriberCount() throws ParsingException {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 500);
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java

@@ -4,6 +4,7 @@
 import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.ExtractorAsserts;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
@@ -51,9 +52,8 @@ public void testGetUploaderName() throws ParsingException {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testGetStreamCount() throws ParsingException {
-            assertEquals(35, extractor.getStreamCount());
+            ExtractorAsserts.assertAtLeast(39, extractor.getStreamCount());
         }
 
         @Test
@@ -62,7 +62,6 @@ public void testGetSubChannelUrl() throws ParsingException {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testGetSubChannelName() throws ParsingException {
             assertEquals("SHOCKING !", extractor.getSubChannelName());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java

@@ -94,13 +94,6 @@ public void testGetLanguageInformation() throws ParsingException {
         @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
         @Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
         @Override public int expectedStreamSegmentsCount() { return 0; }
-
-        @Override
-        @Test
-        @Ignore("TODO fix")
-        public void testSubChannelName() throws Exception {
-            super.testSubChannelName();
-        }
     }
 
     @Ignore("TODO fix")

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -42,7 +42,6 @@ public void testServiceId() {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testName() {
             assertEquals("THE PERFECT LUV TAPE®️", extractor.getName());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java

@@ -26,7 +26,6 @@ public static void setUp() throws Exception {
     }
 
     @Test(expected = IllegalArgumentException.class)
-    @Ignore("TODO fix")
     public void getIdWithNullAsUrl() throws ParsingException {
         linkHandler.fromUrl(null).getId();
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java

@@ -114,7 +114,6 @@ public static void setUp() throws Exception {
 
     public static class PagingTest {
         @Test
-        @Ignore("TODO fix")
         public void duplicatedItemsCheck() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             final SearchExtractor extractor = SoundCloud.getSearchExtractor("cirque du soleil", singletonList(TRACKS), "");

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java

@@ -24,7 +24,6 @@ private static String removeClientId(String url) {
     }
 
     @Test
-    @Ignore("TODO fix")
     public void testRegularValues() throws Exception {
         assertEquals("https://api-v2.soundcloud.com/search?q=asdf&limit=10&offset=0",
                 removeClientId(SoundCloud.getSearchQHFactory().fromQuery("asdf").getUrl()));

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -117,7 +117,6 @@ public void testMoreRelatedItems() throws Exception {
          //////////////////////////////////////////////////////////////////////////*/
 
         @Test
-        @Ignore("TODO fix")
         public void testDescription() throws Exception {
             assertTrue(extractor.getDescription().contains("Zart im Schmelz und süffig im Abgang. Ungebremster Spieltrieb"));
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
@@ -124,6 +125,7 @@ public static void setUp() throws Exception {
         }
 
         @Test
+        @Ignore("TODO fix")
         public void testGetCommentsAllData() throws IOException, ExtractionException {
             final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -53,11 +53,4 @@ public static void setUp() throws Exception {
     @Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
     @Override public long expectedLikeCountAtLeast() { return 13300; }
     @Override public long expectedDislikeCountAtLeast() { return 2600; }
-
-    @Override
-    @Test
-    @Ignore("TODO fix")
-    public void testErrorMessage() throws Exception {
-        super.testErrorMessage();
-    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -268,7 +268,6 @@ public void testStreamSegment() throws Exception {
         }
     }
 
-    @Ignore("TODO fix")
     public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
         private static final String ID = "q6fgbYWsMgw";
         private static final int TIMESTAMP = 0;