Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YouTube] Add fallback to system playlists when fetching age-restricted stream channel tabs directly #1100

Open
wants to merge 5 commits into
base: dev
Choose a base branch
from
Open

[YouTube] Add fallback to system playlists when fetching age-restricted stream channel tabs directly #1100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
155960f
[YouTube] Move channel age-gate renderer extraction method
AudricV Aug 27, 2023
2f2e8a7
[YouTube] Add channel tabs direct fallback to system playlists
AudricV Aug 27, 2023
222823c
[YouTube] Update age restricted channel tabs tests
AudricV Aug 27, 2023
e85144d
[YouTube] Remove useless method overrides and import in channel tab t…
AudricV Aug 27, 2023
2fb7a35
[YouTube] Update age-restricted channel tabs mocks
AudricV Aug 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.schabi.newpipe.extractor.localization.Localization;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
Expand Down Expand Up @@ -320,4 +321,46 @@ public static Optional<ChannelHeader> getChannelHeader(
return Optional.empty();
}
}

/**
* Get a {@code channelAgeGateRenderer} object from a channel response if it exists.
*
* <p>
* A {@code channelAgeGateRenderer} is returned when a channel is age-restricted (creator seems
* to be able to set this setting), its pages are only accessible to logged-in and age-verified
* users. This renderer contains only the following channel metadata: name and avatar.
* </p>
*
* <p>
* This restriction doesn't seem to apply to all countries.
* </p>
*
* <p>
* At most one {@code channelAgeGateRenderer} should be returned per age-restricted channel
* response.
* </p>
*
* @param jsonResponse a channel JSON response
* @return the first {@code channelAgeGateRenderer} if there is one present or {@code null}
*/
@Nullable
public static JsonObject getChannelAgeGateRenderer(@Nonnull final JsonObject jsonResponse) {
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(tab -> tab.getObject("tabRenderer")
.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast))
.filter(content -> content.has("channelAgeGateRenderer"))
.map(content -> content.getObject("channelAgeGateRenderer"))
.findFirst()
.orElse(null);
}
}
32 changes: 1 addition & 31 deletions ...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {

private String channelId;

/**
* If a channel is age-restricted, its pages are only accessible to logged-in and
* age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only
* the following metadata: channel name and channel avatar.
*
* <p>
* This restriction doesn't seem to apply to all countries.
* </p>
*/
@Nullable
private JsonObject channelAgeGateRenderer;

Expand All @@ -95,28 +86,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
jsonResponse = data.jsonResponse;
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
channelId = data.channelId;
channelAgeGateRenderer = getChannelAgeGateRenderer();
}

@Nullable
private JsonObject getChannelAgeGateRenderer() {
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(tab -> tab.getObject("tabRenderer")
.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast))
.filter(content -> content.has("channelAgeGateRenderer"))
.map(content -> content.getObject("channelAgeGateRenderer"))
.findFirst()
.orElse(null);
channelAgeGateRenderer = YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse);
}

@Nonnull
Expand Down
85 changes: 63 additions & 22 deletions .../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,15 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
private String channelId;
@Nullable
private String visitorData;
@Nullable
private JsonObject channelAgeGateRenderer;
@Nullable
private YoutubeChannelTabPlaylistExtractor playlistExtractorInstance;

public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
useVisitorData = getName().equals(ChannelTabs.SHORTS);
useVisitorData = ChannelTabs.SHORTS.equals(getName());
}

@Nonnull
Expand Down Expand Up @@ -101,6 +105,23 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
if (useVisitorData) {
visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
}
channelAgeGateRenderer = YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse);
if (channelAgeGateRenderer != null) {
final String channelTabName = getName();
if (ChannelTabs.VIDEOS.equals(channelTabName)
|| ChannelTabs.SHORTS.equals(channelTabName)
|| ChannelTabs.LIVESTREAMS.equals(channelTabName)) {
Comment on lines +111 to +113
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add these to a list and check if it contains the channelTabName instead?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could, but would this change improve the performance of the checks or not?

final ListLinkHandler originalLinkHandler = getLinkHandler();
playlistExtractorInstance =
new YoutubeChannelTabPlaylistExtractor(getService(),
new ListLinkHandler(originalLinkHandler.getOriginalUrl(),
originalLinkHandler.getUrl(),
getId(),
originalLinkHandler.getContentFilters(),
originalLinkHandler.getSortFilter()));
playlistExtractorInstance.fetchPage();
}
}
}

@Nonnull
Expand All @@ -117,29 +138,31 @@ public String getUrl() throws ParsingException {
@Nonnull
@Override
public String getId() throws ParsingException {
final String id = jsonResponse.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", "");
if (channelAgeGateRenderer == null) {
final String id = jsonResponse.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", "");

if (!id.isEmpty()) {
return id;
}
if (!id.isEmpty()) {
return id;
}

final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.flatMap(item ->
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId")));
if (carouselHeaderId.isPresent()) {
return carouselHeaderId.get();
final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.flatMap(item ->
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId")));
if (carouselHeaderId.isPresent()) {
return carouselHeaderId.get();
}
}

if (!isNullOrEmpty(channelId)) {
Expand All @@ -150,6 +173,10 @@ public String getId() throws ParsingException {
}

protected String getChannelName() {
if (channelAgeGateRenderer != null) {
return channelAgeGateRenderer.getString("channelTitle");
}

final String metadataName = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
Expand All @@ -176,6 +203,13 @@ protected String getChannelName() {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
if (channelAgeGateRenderer != null) {
if (playlistExtractorInstance != null) {
return playlistExtractorInstance.getInitialPage();
}
return InfoItemsPage.emptyPage();
}

final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());

JsonArray items = new JsonArray();
Expand Down Expand Up @@ -223,6 +257,13 @@ public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionEx
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (channelAgeGateRenderer != null) {
if (playlistExtractorInstance != null) {
return playlistExtractorInstance.getPage(page);
}
return InfoItemsPage.emptyPage();
}

if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
Expand Down
74 changes: 37 additions & 37 deletions ...st/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelTabExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;

Expand Down Expand Up @@ -43,7 +42,6 @@ static void setUp() throws IOException, ExtractionException {
@Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q/videos"; }
@Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/user/creativecommons/videos"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@Override public boolean expectedHasMoreItems() { return true; }
}

static class Playlists extends DefaultListExtractorTest<ChannelTabExtractor> {
Expand All @@ -65,7 +63,6 @@ static void setUp() throws IOException, ExtractionException {
@Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ/playlists"; }
@Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/@EEVblog/playlists"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
@Override public boolean expectedHasMoreItems() { return true; }
}

static class Channels extends DefaultListExtractorTest<ChannelTabExtractor> {
Expand All @@ -87,7 +84,6 @@ static void setUp() throws IOException, ExtractionException {
@Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ/channels"; }
@Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ/channels"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
@Override public boolean expectedHasMoreItems() { return true; }
}

static class Livestreams extends DefaultListExtractorTest<ChannelTabExtractor> {
Expand All @@ -109,7 +105,6 @@ static void setUp() throws IOException, ExtractionException {
@Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCR-DXc1voovS8nhAvccRZhg/streams"; }
@Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/c/JeffGeerling/streams"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@Override public boolean expectedHasMoreItems() { return true; }
}

static class Shorts extends DefaultListExtractorTest<ChannelTabExtractor> {
Expand All @@ -131,32 +126,12 @@ static void setUp() throws IOException, ExtractionException {
@Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCh8gHdtzO2tXd593_bjErWg/shorts"; }
@Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/channel/UCh8gHdtzO2tXd593_bjErWg/shorts"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@Override public boolean expectedHasMoreItems() { return true; }
}

private static abstract class AgeRestrictedTabsVideosBaseTest
extends DefaultListExtractorTest<ChannelTabExtractor> {

// TESTS FOR TABS OF AGE RESTRICTED CHANNELS
// Fetching the tabs individually would use the standard tabs without fallback to
// system playlists for stream tabs, we need to fetch the channel extractor to get the
// channel playlist tabs
// TODO: implement system playlists fallback in YoutubeChannelTabExtractor for stream
// tabs

static class AgeRestrictedTabsVideos extends DefaultListExtractorTest<ChannelTabExtractor> {
private static ChannelTabExtractor extractor;

@BeforeAll
static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "ageRestrictedTabsVideos"));
final ChannelExtractor channelExtractor = YouTube.getChannelExtractor(
"https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig");
channelExtractor.fetchPage();

// the videos tab is the first one
extractor = YouTube.getChannelTabExtractor(channelExtractor.getTabs().get(0));
extractor.fetchPage();
}
protected static ChannelTabExtractor extractor;

@Override public ChannelTabExtractor extractor() throws Exception { return extractor; }
@Override public StreamingService expectedService() throws Exception { return YouTube; }
Expand All @@ -165,7 +140,18 @@ static void setUp() throws IOException, ExtractionException {
@Override public String expectedUrlContains() throws Exception { return "https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig/videos"; }
@Override public String expectedOriginalUrlContains() throws Exception { return "https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig/videos"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@Override public boolean expectedHasMoreItems() { return true; }
}

static class AgeRestrictedTabsVideos extends AgeRestrictedTabsVideosBaseTest {
@BeforeAll
static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(
RESOURCE_PATH + "ageRestrictedTabsVideos"));
extractor = YouTube.getChannelTabExtractorFromId(
"channel/UCbfnHqxXs_K3kvaH-WlNlig", ChannelTabs.VIDEOS);
extractor.fetchPage();
}
}

static class AgeRestrictedTabsShorts extends DefaultListExtractorTest<ChannelTabExtractor> {
Expand All @@ -174,13 +160,10 @@ static class AgeRestrictedTabsShorts extends DefaultListExtractorTest<ChannelTab
@BeforeAll
static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "ageRestrictedTabsShorts"));
final ChannelExtractor channelExtractor = YouTube.getChannelExtractor(
"https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig");
channelExtractor.fetchPage();

// the shorts tab is the second one
extractor = YouTube.getChannelTabExtractor(channelExtractor.getTabs().get(1));
NewPipe.init(DownloaderFactory.getDownloader(
RESOURCE_PATH + "ageRestrictedTabsShorts"));
extractor = YouTube.getChannelTabExtractorFromId(
"channel/UCbfnHqxXs_K3kvaH-WlNlig", ChannelTabs.SHORTS);
extractor.fetchPage();
}

Expand All @@ -195,9 +178,26 @@ static void setUp() throws IOException, ExtractionException {
@Test
@Override
public void testRelatedItems() throws Exception {
// this channel has no shorts, so an empty page is returned by the playlist extractor
// This channel has no shorts, so an empty page should be returned by the playlist
// extractor
assertTrue(extractor.getInitialPage().getItems().isEmpty());
assertTrue(extractor.getInitialPage().getErrors().isEmpty());
}
}

static class AgeRestrictedTabsVideosFromChannel extends AgeRestrictedTabsVideosBaseTest {
@BeforeAll
static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(
RESOURCE_PATH + "ageRestrictedTabsVideosFromChannel"));
final ChannelExtractor channelExtractor = YouTube.getChannelExtractor(
"https://www.youtube.com/channel/UCbfnHqxXs_K3kvaH-WlNlig");
channelExtractor.fetchPage();

// the videos tab is the first one
extractor = YouTube.getChannelTabExtractor(channelExtractor.getTabs().get(0));
extractor.fetchPage();
}
}
}
Loading