diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java index 31cfa28565..fc202372a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -320,4 +321,46 @@ public static Optional getChannelHeader( return Optional.empty(); } } + + /** + * Get a {@code channelAgeGateRenderer} object from a channel response if it exists. + * + *

+ * A {@code channelAgeGateRenderer} is returned when a channel is age-restricted (creator seems + * to be able to set this setting), its pages are only accessible to logged-in and age-verified + * users. This renderer contains only the following channel metadata: name and avatar. + *

+ * + *

+ * This restriction doesn't seem to apply to all countries. + *

+ * + *

+ * At most one {@code channelAgeGateRenderer} should be returned per age-restricted channel + * response. + *

+ * + * @param jsonResponse a channel JSON response + * @return the first {@code channelAgeGateRenderer} if there is one present or {@code null} + */ + @Nullable + public static JsonObject getChannelAgeGateRenderer(@Nonnull final JsonObject jsonResponse) { + return jsonResponse.getObject("contents") + .getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .flatMap(tab -> tab.getObject("tabRenderer") + .getObject("content") + .getObject("sectionListRenderer") + .getArray("contents") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast)) + .filter(content -> content.has("channelAgeGateRenderer")) + .map(content -> content.getObject("channelAgeGateRenderer")) + .findFirst() + .orElse(null); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index f35d20ebcc..0c14428813 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -66,15 +66,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private String channelId; - /** - * If a channel is age-restricted, its pages are only accessible to logged-in and - * age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only - * the following metadata: channel name and channel avatar. - * - *

- * This restriction doesn't seem to apply to all countries. - *

- */ @Nullable private JsonObject channelAgeGateRenderer; @@ -95,28 +86,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) jsonResponse = data.jsonResponse; channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse); channelId = data.channelId; - channelAgeGateRenderer = getChannelAgeGateRenderer(); - } - - @Nullable - private JsonObject getChannelAgeGateRenderer() { - return jsonResponse.getObject("contents") - .getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .flatMap(tab -> tab.getObject("tabRenderer") - .getObject("content") - .getObject("sectionListRenderer") - .getArray("contents") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast)) - .filter(content -> content.has("channelAgeGateRenderer")) - .map(content -> content.getObject("channelAgeGateRenderer")) - .findFirst() - .orElse(null); + channelAgeGateRenderer = YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index 27524ee169..efa43e399c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -61,11 +61,15 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { private String channelId; @Nullable private String visitorData; + @Nullable + private JsonObject channelAgeGateRenderer; + @Nullable + private YoutubeChannelTabPlaylistExtractor playlistExtractorInstance; public YoutubeChannelTabExtractor(final StreamingService service, final ListLinkHandler linkHandler) { super(service, linkHandler); - useVisitorData = getName().equals(ChannelTabs.SHORTS); + useVisitorData = ChannelTabs.SHORTS.equals(getName()); } @Nonnull @@ -101,6 +105,23 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException if (useVisitorData) { visitorData = jsonResponse.getObject("responseContext").getString("visitorData"); } + channelAgeGateRenderer = YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse); + if (channelAgeGateRenderer != null) { + final String channelTabName = getName(); + if (ChannelTabs.VIDEOS.equals(channelTabName) + || ChannelTabs.SHORTS.equals(channelTabName) + || ChannelTabs.LIVESTREAMS.equals(channelTabName)) { + final ListLinkHandler originalLinkHandler = getLinkHandler(); + playlistExtractorInstance = + new YoutubeChannelTabPlaylistExtractor(getService(), + new ListLinkHandler(originalLinkHandler.getOriginalUrl(), + originalLinkHandler.getUrl(), + getId(), + originalLinkHandler.getContentFilters(), + originalLinkHandler.getSortFilter())); + playlistExtractorInstance.fetchPage(); + } + } } @Nonnull @@ -117,29 +138,31 @@ public String getUrl() throws ParsingException { @Nonnull @Override public String getId() throws ParsingException { - final String id = jsonResponse.getObject("header") - .getObject("c4TabbedHeaderRenderer") - .getString("channelId", ""); + if (channelAgeGateRenderer == null) { + final String id = jsonResponse.getObject("header") + .getObject("c4TabbedHeaderRenderer") + .getString("channelId", ""); - if (!id.isEmpty()) { - return id; - } + if (!id.isEmpty()) { + return id; + } - final Optional carouselHeaderId = jsonResponse.getObject("header") - .getObject("carouselHeaderRenderer") - .getArray("contents") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .filter(item -> item.has("topicChannelDetailsRenderer")) - .findFirst() - .flatMap(item -> - Optional.ofNullable(item.getObject("topicChannelDetailsRenderer") - .getObject("navigationEndpoint") - .getObject("browseEndpoint") - .getString("browseId"))); - if (carouselHeaderId.isPresent()) { - return carouselHeaderId.get(); + final Optional carouselHeaderId = jsonResponse.getObject("header") + .getObject("carouselHeaderRenderer") + .getArray("contents") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(item -> item.has("topicChannelDetailsRenderer")) + .findFirst() + .flatMap(item -> + Optional.ofNullable(item.getObject("topicChannelDetailsRenderer") + .getObject("navigationEndpoint") + .getObject("browseEndpoint") + .getString("browseId"))); + if (carouselHeaderId.isPresent()) { + return carouselHeaderId.get(); + } } if (!isNullOrEmpty(channelId)) { @@ -150,6 +173,10 @@ public String getId() throws ParsingException { } protected String getChannelName() { + if (channelAgeGateRenderer != null) { + return channelAgeGateRenderer.getString("channelTitle"); + } + final String metadataName = jsonResponse.getObject("metadata") .getObject("channelMetadataRenderer") .getString("title"); @@ -176,6 +203,13 @@ protected String getChannelName() { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + if (channelAgeGateRenderer != null) { + if (playlistExtractorInstance != null) { + return playlistExtractorInstance.getInitialPage(); + } + return InfoItemsPage.emptyPage(); + } + final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); JsonArray items = new JsonArray(); @@ -223,6 +257,13 @@ public InfoItemsPage getInitialPage() throws IOException, ExtractionEx @Override public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + if (channelAgeGateRenderer != null) { + if (playlistExtractorInstance != null) { + return playlistExtractorInstance.getPage(page); + } + return InfoItemsPage.emptyPage(); + } + if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelTabExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelTabExtractorTest.java index ea226a57be..f3f40c6cbc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelTabExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelTabExtractorTest.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -43,7 +42,6 @@ static void setUp() throws IOException, ExtractionException { @Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q/videos"; } @Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/user/creativecommons/videos"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } - @Override public boolean expectedHasMoreItems() { return true; } } static class Playlists extends DefaultListExtractorTest { @@ -65,7 +63,6 @@ static void setUp() throws IOException, ExtractionException { @Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ/playlists"; } @Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/@EEVblog/playlists"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } - @Override public boolean expectedHasMoreItems() { return true; } } static class Channels extends DefaultListExtractorTest { @@ -87,7 +84,6 @@ static void setUp() throws IOException, ExtractionException { @Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ/channels"; } @Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ/channels"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } - @Override public boolean expectedHasMoreItems() { return true; } } static class Livestreams extends DefaultListExtractorTest { @@ -109,7 +105,6 @@ static void setUp() throws IOException, ExtractionException { @Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCR-DXc1voovS8nhAvccRZhg/streams"; } @Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/c/JeffGeerling/streams"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } - @Override public boolean expectedHasMoreItems() { return true; } } static class Shorts extends DefaultListExtractorTest { @@ -131,32 +126,12 @@ static void setUp() throws IOException, ExtractionException { @Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCh8gHdtzO2tXd593_bjErWg/shorts"; } @Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/channel/UCh8gHdtzO2tXd593_bjErWg/shorts"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } - @Override public boolean expectedHasMoreItems() { return true; } } + private static abstract class AgeRestrictedTabsVideosBaseTest + extends DefaultListExtractorTest { - // TESTS FOR TABS OF AGE RESTRICTED CHANNELS - // Fetching the tabs individually would use the standard tabs without fallback to - // system playlists for stream tabs, we need to fetch the channel extractor to get the - // channel playlist tabs - // TODO: implement system playlists fallback in YoutubeChannelTabExtractor for stream - // tabs - - static class AgeRestrictedTabsVideos extends DefaultListExtractorTest { - private static ChannelTabExtractor extractor; - - @BeforeAll - static void setUp() throws IOException, ExtractionException { - YoutubeTestsUtils.ensureStateless(); - NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "ageRestrictedTabsVideos")); - final ChannelExtractor channelExtractor = YouTube.getChannelExtractor( - "https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig"); - channelExtractor.fetchPage(); - - // the videos tab is the first one - extractor = YouTube.getChannelTabExtractor(channelExtractor.getTabs().get(0)); - extractor.fetchPage(); - } + protected static ChannelTabExtractor extractor; @Override public ChannelTabExtractor extractor() throws Exception { return extractor; } @Override public StreamingService expectedService() throws Exception { return YouTube; } @@ -165,7 +140,18 @@ static void setUp() throws IOException, ExtractionException { @Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig/videos"; } @Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig/videos"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } - @Override public boolean expectedHasMoreItems() { return true; } + } + + static class AgeRestrictedTabsVideos extends AgeRestrictedTabsVideosBaseTest { + @BeforeAll + static void setUp() throws IOException, ExtractionException { + YoutubeTestsUtils.ensureStateless(); + NewPipe.init(DownloaderFactory.getDownloader( + RESOURCE_PATH + "ageRestrictedTabsVideos")); + extractor = YouTube.getChannelTabExtractorFromId( + "channel/UCbfnHqxXs_K3kvaH-WlNlig", ChannelTabs.VIDEOS); + extractor.fetchPage(); + } } static class AgeRestrictedTabsShorts extends DefaultListExtractorTest { @@ -174,13 +160,10 @@ static class AgeRestrictedTabsShorts extends DefaultListExtractorTest