Merge pull request #620 from TeamNewPipe/bandcamp-comments

Add Bandcamp comments (first page)

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java

@@ -16,9 +16,11 @@
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
+import java.util.Arrays;
 import java.util.Collections;
 
 import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
+import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL;
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED;
@@ -28,7 +30,7 @@
 public class BandcampService extends StreamingService {
 
     public BandcampService(final int id) {
-        super(id, "Bandcamp", Collections.singletonList(AUDIO));
+        super(id, "Bandcamp", Arrays.asList(AUDIO, COMMENTS));
     }
 
     @Override
@@ -58,7 +60,7 @@ public SearchQueryHandlerFactory getSearchQHFactory() {
 
     @Override
     public ListLinkHandlerFactory getCommentsLHFactory() {
-        return null;
+        return new BandcampCommentsLinkHandlerFactory();
     }
 
     @Override
@@ -122,6 +124,6 @@ public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
 
     @Override
     public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) {
-        return null;
+        return new BandcampCommentsExtractor(this, linkHandler);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsExtractor.java

@@ -0,0 +1,53 @@
+package org.schabi.newpipe.extractor.services.bandcamp.extractors;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.schabi.newpipe.extractor.Page;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.comments.CommentsExtractor;
+import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
+import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+public class BandcampCommentsExtractor extends CommentsExtractor {
+
+    private Document document;
+
+
+    public BandcampCommentsExtractor(StreamingService service, ListLinkHandler linkHandler) {
+        super(service, linkHandler);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        String html = downloader.get(getLinkHandler().getUrl()).responseBody();
+        document = Jsoup.parse(html);
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
+
+        CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
+
+        Elements writings = document.getElementsByClass("writing");
+
+        for (Element writing : writings) {
+            collector.commit(new BandcampCommentsInfoItemExtractor(writing, getUrl()));
+        }
+
+        return new InfoItemsPage<>(collector, null);
+    }
+
+    @Override
+    public InfoItemsPage<CommentsInfoItem> getPage(Page page) throws IOException, ExtractionException {
+        return null;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java

@@ -0,0 +1,92 @@
+package org.schabi.newpipe.extractor.services.bandcamp.extractors;
+
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.localization.DateWrapper;
+
+import javax.annotation.Nullable;
+
+public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
+
+    private final Element writing;
+    private final String url;
+
+    public BandcampCommentsInfoItemExtractor(Element writing, String url) {
+        this.writing = writing;
+        this.url = url;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return writing.getElementsByClass("text").first().ownText();
+    }
+
+    @Override
+    public String getUrl() {
+        return url;
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        return writing.getElementsByClass("thumb").attr("src");
+    }
+
+    @Override
+    public int getLikeCount() {
+        return -1;
+    }
+
+    @Override
+    public String getCommentText() {
+        return writing.getElementsByClass("text").first().ownText();
+    }
+
+    @Override
+    public String getTextualUploadDate() {
+        return "";
+    }
+
+    @Nullable
+    @Override
+    public DateWrapper getUploadDate() {
+        return null;
+    }
+
+    @Override
+    public String getCommentId() {
+        return "";
+    }
+
+    @Override
+    public String getUploaderUrl() {
+        //return writing.getElementsByClass("name").attr("href");
+        // Fan links cannot be opened
+        return "";
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        return writing.getElementsByClass("name").first().text();
+    }
+
+    @Override
+    public String getUploaderAvatarUrl() {
+        return writing.getElementsByClass("thumb").attr("src");
+    }
+
+    @Override
+    public boolean isHeartedByUploader() {
+        return false;
+    }
+
+    @Override
+    public boolean isPinned() {
+        return false;
+    }
+
+    @Override
+    public boolean isUploaderVerified() throws ParsingException {
+        return false;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampCommentsLinkHandlerFactory.java

@@ -0,0 +1,33 @@
+package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+
+import java.util.List;
+
+/**
+ * Like in {@link BandcampStreamLinkHandlerFactory}, tracks have no meaningful IDs except for
+ * their URLs
+ */
+public class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
+
+    @Override
+    public String getId(String url) throws ParsingException {
+        return url;
+    }
+
+    @Override
+    public boolean onAcceptUrl(String url) throws ParsingException {
+        // Don't accept URLs that don't point to a track
+        if (!url.toLowerCase().matches("https?://.+\\..+/(track|album)/.+")) return false;
+
+        // Test whether domain is supported
+        return BandcampExtractorHelper.isSupportedDomain(url);
+    }
+
+    @Override
+    public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
+        return id;
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java

@@ -0,0 +1,52 @@
+package org.schabi.newpipe.extractor.services.bandcamp;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.comments.CommentsExtractor;
+import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.services.DefaultTests;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertFalse;
+import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+public class BandcampCommentsExtractorTest {
+
+    private static CommentsExtractor extractor;
+
+    @BeforeClass
+    public static void setUp() throws ExtractionException, IOException {
+        NewPipe.init(DownloaderTestImpl.getInstance());
+        extractor = Bandcamp.getCommentsExtractor("https://floatingpoints.bandcamp.com/album/promises");
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void hasComments() throws IOException, ExtractionException {
+        assertTrue(extractor.getInitialPage().getItems().size() >= 3);
+    }
+
+    @Test
+    public void testGetCommentsAllData() throws IOException, ExtractionException {
+        ListExtractor.InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
+
+        DefaultTests.defaultTestListOfItems(Bandcamp, comments.getItems(), comments.getErrors());
+        for (CommentsInfoItem c : comments.getItems()) {
+            assertFalse(Utils.isBlank(c.getUploaderName()));
+            assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
+            assertFalse(Utils.isBlank(c.getCommentText()));
+            assertFalse(Utils.isBlank(c.getName()));
+            assertFalse(Utils.isBlank(c.getThumbnailUrl()));
+            assertFalse(Utils.isBlank(c.getUrl()));
+            assertEquals(-1, c.getLikeCount());
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsLinkHandlerFactoryTest.java

@@ -0,0 +1,45 @@
+// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
+
+package org.schabi.newpipe.extractor.services.bandcamp;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampCommentsLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test for {@link BandcampCommentsLinkHandlerFactory}
+ */
+public class BandcampCommentsLinkHandlerFactoryTest {
+
+    private static BandcampCommentsLinkHandlerFactory linkHandler;
+
+    @BeforeClass
+    public static void setUp() {
+        linkHandler = new BandcampCommentsLinkHandlerFactory();
+        NewPipe.init(DownloaderTestImpl.getInstance());
+    }
+
+    @Test
+    public void testAcceptUrl() throws ParsingException {
+        assertFalse(linkHandler.acceptUrl("http://interovgm.com/releases/"));
+        assertFalse(linkHandler.acceptUrl("https://interovgm.com/releases"));
+        assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com"));
+        assertFalse(linkHandler.acceptUrl("https://bandcamp.com"));
+        assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/"));
+        assertFalse(linkHandler.acceptUrl("https://example.com/track/sampletrack"));
+        assertFalse(linkHandler.acceptUrl("http://bandcamP.com/?show=38"));
+
+        assertTrue(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids"));
+        assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen"));
+        assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/"));
+        assertTrue(linkHandler.acceptUrl("https://interovgm.bandcamp.com/track/title"));
+        assertTrue(linkHandler.acceptUrl("https://goodgoodblood-tl.bandcamp.com/track/when-it-all-wakes-up"));
+        assertTrue(linkHandler.acceptUrl("https://lobstertheremin.com/track/unfinished"));
+    }
+}