Merge pull request #314 from wb9688/remove-getnextpageurl

Next page stuff
TeamNewPipe · Jul 7, 2020 · a70cb02 · a70cb02
2 parents e5d23a8 + 0a5a905
commit a70cb02
Show file tree

Hide file tree

Showing 53 changed files with 778 additions and 932 deletions.
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
@@ -2,21 +2,18 @@
 
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 
 import javax.annotation.Nonnull;
 
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /**
  * Base class to extractors that have a list (e.g. playlists, users).
  */
 public abstract class ListExtractor<R extends InfoItem> extends Extractor {
-
     /**
      * Constant that should be returned whenever
      * a list has an unknown number of items.
@@ -38,36 +35,22 @@ public ListExtractor(StreamingService service, ListLinkHandler linkHandler) {
     }
 
     /**
-     * A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page where the items are from the initial request and
-     * the nextPageUrl relative to it.
+     * A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page
+     * where the items are from the initial request and the nextPage relative to it.
      *
      * @return a {@link InfoItemsPage} corresponding to the initial page
      */
     @Nonnull
     public abstract InfoItemsPage<R> getInitialPage() throws IOException, ExtractionException;
 
-    /**
-     * Returns an url that can be used to get the next page relative to the initial one.
-     * <p>Usually, these links will only work in the implementation itself.</p>
-     *
-     * @return an url pointing to the next page relative to the initial page
-     * @see #getPage(String)
-     */
-    public abstract String getNextPageUrl() throws IOException, ExtractionException;
-
     /**
      * Get a list of items corresponding to the specific requested page.
      *
-     * @param pageUrl any page url got from the exclusive implementation of the list extractor
+     * @param page any page got from the exclusive implementation of the list extractor
      * @return a {@link InfoItemsPage} corresponding to the requested page
-     * @see #getNextPageUrl()
-     * @see InfoItemsPage#getNextPageUrl()
+     * @see InfoItemsPage#getNextPage()
      */
-    public abstract InfoItemsPage<R> getPage(final String pageUrl) throws IOException, ExtractionException;
-
-    public boolean hasNextPage() throws IOException, ExtractionException {
-        return !isNullOrEmpty(getNextPageUrl());
-    }
+    public abstract InfoItemsPage<R> getPage(final Page page) throws IOException, ExtractionException;
 
     @Override
     public ListLinkHandler getLinkHandler() {
@@ -80,23 +63,22 @@ public ListLinkHandler getLinkHandler() {
 
     /**
      * A class that is used to wrap a list of gathered items and eventual errors, it
-     * also contains a field that points to the next available page ({@link #nextPageUrl}).
+     * also contains a field that points to the next available page ({@link #nextPage}).
      */
     public static class InfoItemsPage<T extends InfoItem> {
         private static final InfoItemsPage<InfoItem> EMPTY =
-                new InfoItemsPage<>(Collections.<InfoItem>emptyList(), "", Collections.<Throwable>emptyList());
+                new InfoItemsPage<>(Collections.<InfoItem>emptyList(), null, Collections.<Throwable>emptyList());
 
         /**
          * A convenient method that returns a representation of an empty page.
          *
-         * @return a type-safe page with the list of items and errors empty and the nextPageUrl set to an empty string.
+         * @return a type-safe page with the list of items and errors empty and the nextPage set to {@code null}.
          */
         public static <T extends InfoItem> InfoItemsPage<T> emptyPage() {
             //noinspection unchecked
             return (InfoItemsPage<T>) EMPTY;
         }
 
-
         /**
          * The current list of items of this page
          */
@@ -105,40 +87,40 @@ public static <T extends InfoItem> InfoItemsPage<T> emptyPage() {
         /**
          * Url pointing to the next page relative to this one
          *
-         * @see ListExtractor#getPage(String)
+         * @see ListExtractor#getPage(Page)
+         * @see Page
          */
-        private final String nextPageUrl;
+        private final Page nextPage;
 
         /**
          * Errors that happened during the extraction
          */
         private final List<Throwable> errors;
 
-        public InfoItemsPage(InfoItemsCollector<T, ?> collector, String nextPageUrl) {
-            this(collector.getItems(), nextPageUrl, collector.getErrors());
+        public InfoItemsPage(InfoItemsCollector<T, ?> collector, Page nextPage) {
+            this(collector.getItems(), nextPage, collector.getErrors());
         }
 
-        public InfoItemsPage(List<T> itemsList, String nextPageUrl, List<Throwable> errors) {
+        public InfoItemsPage(List<T> itemsList, Page nextPage, List<Throwable> errors) {
             this.itemsList = itemsList;
-            this.nextPageUrl = nextPageUrl;
+            this.nextPage = nextPage;
             this.errors = errors;
         }
 
         public boolean hasNextPage() {
-            return !isNullOrEmpty(nextPageUrl);
+            return Page.isValid(nextPage);
         }
 
         public List<T> getItems() {
             return itemsList;
         }
 
-        public String getNextPageUrl() {
-            return nextPageUrl;
+        public Page getNextPage() {
+            return nextPage;
         }
 
         public List<Throwable> getErrors() {
             return errors;
         }
     }
-
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java
@@ -4,11 +4,9 @@
 
 import java.util.List;
 
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
 public abstract class ListInfo<T extends InfoItem> extends Info {
     private List<T> relatedItems;
-    private String nextPageUrl = null;
+    private Page nextPage = null;
     private final List<String> contentFilters;
     private final String sortFilter;
 
@@ -39,15 +37,15 @@ public void setRelatedItems(List<T> relatedItems) {
     }
 
     public boolean hasNextPage() {
-        return !isNullOrEmpty(nextPageUrl);
+        return Page.isValid(nextPage);
     }
 
-    public String getNextPageUrl() {
-        return nextPageUrl;
+    public Page getNextPage() {
+        return nextPage;
     }
 
-    public void setNextPageUrl(String pageUrl) {
-        this.nextPageUrl = pageUrl;
+    public void setNextPage(Page page) {
+        this.nextPage = page;
     }
 
     public List<String> getContentFilters() {

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java
@@ -0,0 +1,52 @@
+package org.schabi.newpipe.extractor;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+public class Page implements Serializable {
+    private final String url;
+    private final List<String> ids;
+    private final Map<String, String> cookies;
+
+    public Page(final String url, final List<String> ids, final Map<String, String> cookies) {
+        this.url = url;
+        this.ids = ids;
+        this.cookies = cookies;
+    }
+
+    public Page(final String url) {
+        this(url, null, null);
+    }
+
+    public Page(final String url, final Map<String, String> cookies) {
+        this(url, null, cookies);
+    }
+
+    public Page(final List<String> ids) {
+        this(null, ids, null);
+    }
+
+    public Page(final List<String> ids, final Map<String, String> cookies) {
+        this(null, ids, cookies);
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public List<String> getIds() {
+        return ids;
+    }
+
+    public Map<String, String> getCookies() {
+        return cookies;
+    }
+
+    public static boolean isValid(final Page page) {
+        return page != null && (!isNullOrEmpty(page.getUrl())
+                || !isNullOrEmpty(page.getIds()));
+    }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
@@ -3,6 +3,7 @@
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -49,8 +50,8 @@ public static ChannelInfo getInfo(StreamingService service, String url) throws I
 
     public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
                                                              String url,
-                                                             String pageUrl) throws IOException, ExtractionException {
-        return service.getChannelExtractor(url).getPage(pageUrl);
+                                                             Page page) throws IOException, ExtractionException {
+        return service.getChannelExtractor(url).getPage(page);
     }
 
     public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
@@ -81,7 +82,7 @@ public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException
 
         final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
         info.setRelatedItems(itemsPage.getItems());
-        info.setNextPageUrl(itemsPage.getNextPageUrl());
+        info.setNextPage(itemsPage.getNextPage());
 
         try {
             info.setSubscriberCount(extractor.getSubscriberCount());

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java
@@ -3,6 +3,7 @@
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -39,23 +40,23 @@ private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws
         InfoItemsPage<CommentsInfoItem> initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(commentsInfo,
                 commentsExtractor);
         commentsInfo.setRelatedItems(initialCommentsPage.getItems());
-        commentsInfo.setNextPageUrl(initialCommentsPage.getNextPageUrl());
+        commentsInfo.setNextPage(initialCommentsPage.getNextPage());
 
         return commentsInfo;
     }
 
-    public static InfoItemsPage<CommentsInfoItem> getMoreItems(CommentsInfo commentsInfo, String pageUrl)
+    public static InfoItemsPage<CommentsInfoItem> getMoreItems(CommentsInfo commentsInfo, Page page)
             throws ExtractionException, IOException {
-        return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl);
+        return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, page);
     }
 
     public static InfoItemsPage<CommentsInfoItem> getMoreItems(StreamingService service, CommentsInfo commentsInfo,
-                                                               String pageUrl) throws IOException, ExtractionException {
+                                                               Page page) throws IOException, ExtractionException {
         if (null == commentsInfo.getCommentsExtractor()) {
             commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
             commentsInfo.getCommentsExtractor().fetchPage();
         }
-        return commentsInfo.getCommentsExtractor().getPage(pageUrl);
+        return commentsInfo.getCommentsExtractor().getPage(page);
     }
 
     private transient CommentsExtractor commentsExtractor;

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java
@@ -45,7 +45,7 @@ public static FeedInfo getInfo(FeedExtractor extractor) throws IOException, Extr
 
         final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
         info.setRelatedItems(itemsPage.getItems());
-        info.setNextPageUrl(itemsPage.getNextPageUrl());
+        info.setNextPage(itemsPage.getNextPage());
 
         return info;
     }

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
@@ -23,6 +23,7 @@
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -33,18 +34,17 @@
 import java.io.IOException;
 
 public class KioskInfo extends ListInfo<StreamInfoItem> {
-
     private KioskInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException {
         super(serviceId, linkHandler, name);
     }
 
     public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
                                                                            String url,
-                                                                           String pageUrl)
+                                                                           Page page)
             throws IOException, ExtractionException {
         KioskList kl = service.getKioskList();
-        KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
-        return extractor.getPage(pageUrl);
+        KioskExtractor extractor = kl.getExtractorByUrl(url, page);
+        return extractor.getPage(page);
     }
 
     public static KioskInfo getInfo(String url) throws IOException, ExtractionException {
@@ -71,7 +71,7 @@ public static KioskInfo getInfo(KioskExtractor extractor) throws ExtractionExcep
 
         final ListExtractor.InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
         info.setRelatedItems(itemsPage.getItems());
-        info.setNextPageUrl(itemsPage.getNextPageUrl());
+        info.setNextPage(itemsPage.getNextPage());
 
         return info;
     }