Merge branch 'dev'

TeamNewPipe · Feb 21, 2020 · 4680928 · 4680928
2 parents 33961b2 + 4bc5b8d
commit 4680928
Show file tree

Hide file tree

Showing 130 changed files with 1,895 additions and 1,101 deletions.
diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
-     
+
 > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
 
 ## Supported sites

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java
@@ -1,10 +1,5 @@
 package org.schabi.newpipe.extractor;
 
-import java.io.IOException;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -13,27 +8,33 @@
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 
-public abstract class Extractor{ 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+
+public abstract class Extractor {
     /**
      * {@link StreamingService} currently related to this extractor.<br>
      * Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
      */
     private final StreamingService service;
     private final LinkHandler linkHandler;
 
-    @Nullable private Localization forcedLocalization = null;
-    @Nullable private ContentCountry forcedContentCountry = null;
+    @Nullable
+    private Localization forcedLocalization = null;
+    @Nullable
+    private ContentCountry forcedContentCountry = null;
 
     private boolean pageFetched = false;
     private final Downloader downloader;
 
     public Extractor(final StreamingService service, final LinkHandler linkHandler) {
-        if(service == null) throw new NullPointerException("service is null");
-        if(linkHandler == null) throw new NullPointerException("LinkHandler is null");
+        if (service == null) throw new NullPointerException("service is null");
+        if (linkHandler == null) throw new NullPointerException("LinkHandler is null");
         this.service = service;
         this.linkHandler = linkHandler;
         this.downloader = NewPipe.getDownloader();
-        if(downloader == null) throw new NullPointerException("downloader is null");
+        if (downloader == null) throw new NullPointerException("downloader is null");
     }
 
     /**
@@ -46,11 +47,12 @@ public LinkHandler getLinkHandler() {
 
     /**
      * Fetch the current page.
-     * @throws IOException if the page can not be loaded
+     *
+     * @throws IOException         if the page can not be loaded
      * @throws ExtractionException if the pages content is not understood
      */
     public void fetchPage() throws IOException, ExtractionException {
-        if(pageFetched) return;
+        if (pageFetched) return;
         onFetchPage(downloader);
         pageFetched = true;
     }
@@ -65,8 +67,9 @@ protected boolean isPageFetched() {
 
     /**
      * Fetch the current page.
+     *
      * @param downloader the download to use
-     * @throws IOException if the page can not be loaded
+     * @throws IOException         if the page can not be loaded
      * @throws ExtractionException if the pages content is not understood
      */
     public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
@@ -78,6 +81,7 @@ public String getId() throws ParsingException {
 
     /**
      * Get the name
+     *
      * @return the name
      * @throws ParsingException if the name cannot be extracted
      */
@@ -93,10 +97,10 @@ public String getOriginalUrl() throws ParsingException {
     public String getUrl() throws ParsingException {
         return linkHandler.getUrl();
     }
-   
+
     @Nonnull
     public String getBaseUrl() throws ParsingException {
-       return linkHandler.getBaseUrl();
+        return linkHandler.getBaseUrl();
     }
 
     @Nonnull

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java
@@ -27,7 +27,7 @@
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-public abstract class InfoItemsCollector<I extends InfoItem, E extends InfoItemExtractor> implements Collector<I,E> {
+public abstract class InfoItemsCollector<I extends InfoItem, E extends InfoItemExtractor> implements Collector<I, E> {
 
     private final List<I> itemList = new ArrayList<>();
     private final List<Throwable> errors = new ArrayList<>();

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java
@@ -115,26 +115,28 @@ public static MediaFormat getFromMimeType(String mimeType) {
     }
 
     /**
-     * Get the media format by it's id.
+     * Get the media format by its id.
+     *
      * @param id the id
      * @return the id of the media format or null.
      */
     public static MediaFormat getFormatById(int id) {
-        for (MediaFormat vf: values()) {
+        for (MediaFormat vf : values()) {
             if (vf.id == id) return vf;
         }
         return null;
     }
 
     public static MediaFormat getFromSuffix(String suffix) {
-        for (MediaFormat vf: values()) {
+        for (MediaFormat vf : values()) {
             if (vf.suffix.equals(suffix)) return vf;
         }
         return null;
     }
-    
+
     /**
      * Get the name of the format
+     *
      * @return the name of the format
      */
     public String getName() {
@@ -143,6 +145,7 @@ public String getName() {
 
     /**
      * Get the filename extension
+     *
      * @return the filename extension
      */
     public String getSuffix() {
@@ -151,10 +154,11 @@ public String getSuffix() {
 
     /**
      * Get the mime type
+     *
      * @return the mime type
      */
     public String getMimeType() {
         return mimeType;
     }
-    
+
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java
@@ -1,14 +1,14 @@
 package org.schabi.newpipe.extractor;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeService;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
 /*
  * Copyright (C) Christian Schabesberger 2018 <[email protected]>
  * ServiceList.java is part of NewPipe.

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
@@ -1,19 +1,12 @@
 package org.schabi.newpipe.extractor;
 
-import java.util.Collections;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.feed.FeedExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
+import org.schabi.newpipe.extractor.linkhandler.*;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@@ -24,6 +17,10 @@
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
+
 /*
  * Copyright (C) Christian Schabesberger 2018 <[email protected]>
  * StreamingService.java is part of NewPipe.
@@ -65,7 +62,7 @@ public ServiceInfo(String name, List<MediaCapability> mediaCapabilities) {
         public String getName() {
             return name;
         }
-        
+
         public List<MediaCapability> getMediaCapabilities() {
             return mediaCapabilities;
         }
@@ -116,7 +113,7 @@ public ServiceInfo getServiceInfo() {
     public String toString() {
         return serviceId + ":" + serviceInfo.getName();
     }
-    
+
     public abstract String getBaseUrl();
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -173,6 +170,19 @@ public String toString() {
      */
     public abstract SubscriptionExtractor getSubscriptionExtractor();
 
+    /**
+     * This method decides which strategy will be chosen to fetch the feed. In YouTube, for example, a separate feed
+     * exists which is lightweight and made specifically to be used like this.
+     * <p>
+     * In services which there's no other way to retrieve them, null should be returned.
+     *
+     * @return a {@link FeedExtractor} instance or null.
+     */
+    @Nullable
+    public FeedExtractor getFeedExtractor(String url) throws ExtractionException {
+        return null;
+    }
+
     /**
      * Must create a new instance of a KioskList implementation.
      * @return a new KioskList instance
@@ -253,12 +263,12 @@ public StreamExtractor getStreamExtractor(String url) throws ExtractionException
 
     public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {
         ListLinkHandlerFactory llhf = getCommentsLHFactory();
-        if(null == llhf) {
+        if (llhf == null) {
             return null;
         }
         return getCommentsExtractor(llhf.fromUrl(url));
     }
-    
+
     /*//////////////////////////////////////////////////////////////////////////
     // Utils
     //////////////////////////////////////////////////////////////////////////*/

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
@@ -5,9 +5,7 @@
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
@@ -35,8 +33,8 @@
 
 public class ChannelInfo extends ListInfo<StreamInfoItem> {
 
-    public ChannelInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException {
-        super(serviceId, linkHandler, name);
+    public ChannelInfo(int serviceId, String id, String url, String originalUrl, String name, ListLinkHandler listLinkHandler) {
+        super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(), listLinkHandler.getSortFilter());
     }
 
     public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@@ -57,15 +55,14 @@ public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService servic
 
     public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
 
-        ChannelInfo info = new ChannelInfo(extractor.getServiceId(),
-                extractor.getLinkHandler(),
-                extractor.getName());
+        final int serviceId = extractor.getServiceId();
+        final String id = extractor.getId();
+        final String url = extractor.getUrl();
+        final String originalUrl = extractor.getOriginalUrl();
+        final String name = extractor.getName();
+
+        final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler());
 
-        try {
-            info.setOriginalUrl(extractor.getOriginalUrl());
-        } catch (Exception e) {
-            info.addError(e);
-        }
         try {
             info.setAvatarUrl(extractor.getAvatarUrl());
         } catch (Exception e) {

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java
@@ -33,7 +33,7 @@ public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws Parsin
         // important information
         int serviceId = getServiceId();
         String name = extractor.getName();
-        String  url = extractor.getUrl();
+        String url = extractor.getUrl();
 
         ChannelInfoItem resultItem = new ChannelInfoItem(serviceId, url, name);
 

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java
@@ -1,7 +1,5 @@
 package org.schabi.newpipe.extractor.comments;
 
-import java.io.IOException;
-
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -10,20 +8,21 @@
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
-public class CommentsInfo extends ListInfo<CommentsInfoItem>{
+import java.io.IOException;
+
+public class CommentsInfo extends ListInfo<CommentsInfoItem> {
+
+    private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) {
+        super(serviceId, listUrlIdHandler, name);
+    }
 
-	private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) {
-		super(serviceId, listUrlIdHandler, name);
-		// TODO Auto-generated constructor stub
-	}
-
-	public static CommentsInfo getInfo(String url) throws IOException, ExtractionException {
+    public static CommentsInfo getInfo(String url) throws IOException, ExtractionException {
         return getInfo(NewPipe.getServiceByUrl(url), url);
     }
 
-	public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException {
-	    return getInfo(serviceByUrl.getCommentsExtractor(url));
-	}
+    public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException {
+        return getInfo(serviceByUrl.getCommentsExtractor(url));
+    }
 
     private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException {
         // for services which do not have a comments extractor
@@ -44,21 +43,21 @@ private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws
 
         return commentsInfo;
     }
-    
+
     public static InfoItemsPage<CommentsInfoItem> getMoreItems(CommentsInfo commentsInfo, String pageUrl)
             throws ExtractionException, IOException {
         return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl);
     }
-    
+
     public static InfoItemsPage<CommentsInfoItem> getMoreItems(StreamingService service, CommentsInfo commentsInfo,
-            String pageUrl) throws IOException, ExtractionException {
+                                                               String pageUrl) throws IOException, ExtractionException {
         if (null == commentsInfo.getCommentsExtractor()) {
             commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
             commentsInfo.getCommentsExtractor().fetchPage();
         }
         return commentsInfo.getCommentsExtractor().getPage(pageUrl);
     }
-    
+
     private transient CommentsExtractor commentsExtractor;
 
     public CommentsExtractor getCommentsExtractor() {