From 601773011742ba07b7d7d602d322a8fc1cc1b1be Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 18 Jun 2022 16:07:32 +0200 Subject: [PATCH] [YouTube] Send Content-Type header in all POST requests This header was not sent partially before and was added and guessed by OkHttp. This can create issues when using other HTTP clients than OkHttp, such as Cronet. Some code in the modified classes has been improved and / or deduplicated, and usages of the UTF_8 constant of the Utils class has been replaced by StandardCharsets.UTF_8 where possible. Note that this header has been not added in except in YoutubeDashManifestCreatorsUtils, as an empty body is sent in the POST requests made by this class. --- .../youtube/YoutubeParsingHelper.java | 46 ++++-- .../YoutubeDashManifestCreatorsUtils.java | 4 +- .../extractors/YoutubeChannelExtractor.java | 154 +++++++++--------- .../YoutubeMixPlaylistExtractor.java | 26 ++- .../YoutubeMusicSearchExtractor.java | 29 +--- .../extractors/YoutubePlaylistExtractor.java | 18 +- .../extractors/YoutubeSearchExtractor.java | 93 +++++------ 7 files changed, 186 insertions(+), 184 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 566da52170..f325a6ecce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -591,13 +591,14 @@ public static boolean areHardcodedClientVersionAndKeyValid() .end() .value("fetchLiveState", true) .end() - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_CLIENT_VERSION)); + headers.put("Content-Type", Collections.singletonList("application/json")); // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) @@ -817,16 +818,16 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException, .end() .end() .value("input", "") - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList( - HARDCODED_YOUTUBE_MUSIC_KEY[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList( - HARDCODED_YOUTUBE_MUSIC_KEY[2])); + headers.put("X-YouTube-Client-Name", + Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[1])); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[2])); headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Referer", Collections.singletonList("https://music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); final Response response = getDownloader().post(url, headers, json); @@ -1065,13 +1066,12 @@ public static JsonObject getJsonPostResponse(final String endpoint, final Localization localization) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + addYouTubeHeaders(headers); headers.put("Content-Type", Collections.singletonList("application/json")); - final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" - + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization); - - return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" + getKey() + + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } public static JsonObject getJsonAndroidPostResponse( @@ -1100,17 +1100,18 @@ private static JsonObject getMobilePostResponse( @Nonnull final String innerTubeApiKey, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); - headers.put("Content-Type", Collections.singletonList("application/json")); headers.put("User-Agent", Collections.singletonList(userAgent)); headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2")); + headers.put("Content-Type", Collections.singletonList("application/json")); final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey + DISABLE_PRETTY_PRINT_PARAMETER; - final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) - ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest, - headers, body, localization); - return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) + ? baseEndpointUrl + : baseEndpointUrl + endPartOfUrlRequest, + headers, body, localization))); } @Nonnull @@ -1288,6 +1289,17 @@ public static String getIosUserAgent(@Nullable final Localization localization) + ")"; } + @Nonnull + public static Map> getYoutubeMusicHeaders() { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + return headers; + } + /** * Add required headers and cookies to an existing headers Map. * @see #addClientInfoHeaders(Map) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 5c45f65df0..425e5893af 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -35,7 +35,6 @@ import java.util.Map; import java.util.Objects; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; @@ -708,7 +707,8 @@ private static Response getStreamingWebUrlWithoutRedirects( throws CreationException { try { final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + headers.put("Origin", Collections.singletonList("https://www.youtube.com")); + headers.put("Referer", Collections.singletonList("https://www.youtube.com")); String responseMimeType = ""; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 2318ddc179..6973f81831 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,15 +2,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -21,7 +18,6 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -32,14 +28,13 @@ import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -86,8 +81,8 @@ public YoutubeChannelExtractor(final StreamingService service, } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, - ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { final String channelPath = super.getId(); final String[] channelId = channelPath.split("/"); String id = ""; @@ -99,22 +94,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException getExtractorLocalization(), getExtractorContentCountry()) .value("url", "https://www.youtube.com/" + channelPath) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url", body, getExtractorLocalization()); - if (!isNullOrEmpty(jsonResponse.getObject("error"))) { - final JsonObject errorJsonObject = jsonResponse.getObject("error"); - final int errorCode = errorJsonObject.getInt("code"); - if (errorCode == 404) { - throw new ContentNotAvailableException("This channel doesn't exist."); - } else { - throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); - } - } + checkIfChannelResponseIsValid(jsonResponse); final JsonObject endpoint = jsonResponse.getObject("endpoint"); @@ -147,22 +132,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException .value("browseId", id) .value("params", "EgZ2aWRlb3M%3D") // Equal to videos .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); final JsonObject jsonResponse = getJsonPostResponse("browse", body, getExtractorLocalization()); - if (!isNullOrEmpty(jsonResponse.getObject("error"))) { - final JsonObject errorJsonObject = jsonResponse.getObject("error"); - final int errorCode = errorJsonObject.getInt("code"); - if (errorCode == 404) { - throw new ContentNotAvailableException("This channel doesn't exist."); - } else { - throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); - } - } + checkIfChannelResponseIsValid(jsonResponse); final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions") .getObject(0) @@ -200,6 +175,21 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException YoutubeParsingHelper.defaultAlertsCheck(initialData); } + private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse) + throws ContentNotAvailableException { + if (!isNullOrEmpty(jsonResponse.getObject("error"))) { + final JsonObject errorJsonObject = jsonResponse.getObject("error"); + final int errorCode = errorJsonObject.getInt("code"); + if (errorCode == 404) { + throw new ContentNotAvailableException("This channel doesn't exist."); + } else { + throw new ContentNotAvailableException("Got error:\"" + + errorJsonObject.getString("status") + "\": " + + errorJsonObject.getString("message")); + } + } + } + @Nonnull @Override public String getUrl() throws ParsingException { @@ -230,7 +220,8 @@ public String getId() throws ParsingException { @Override public String getName() throws ParsingException { try { - return initialData.getObject("header").getObject("c4TabbedHeaderRenderer") + return initialData.getObject("header") + .getObject("c4TabbedHeaderRenderer") .getString("title"); } catch (final Exception e) { throw new ParsingException("Could not get channel name", e); @@ -241,8 +232,11 @@ public String getName() throws ParsingException { public String getAvatarUrl() throws ParsingException { try { final String url = initialData.getObject("header") - .getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails") - .getObject(0).getString("url"); + .getObject("c4TabbedHeaderRenderer") + .getObject("avatar") + .getArray("thumbnails") + .getObject(0) + .getString("url"); return fixThumbnailUrl(url); } catch (final Exception e) { @@ -254,8 +248,11 @@ public String getAvatarUrl() throws ParsingException { public String getBannerUrl() throws ParsingException { try { final String url = initialData.getObject("header") - .getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails") - .getObject(0).getString("url"); + .getObject("c4TabbedHeaderRenderer") + .getObject("banner") + .getArray("thumbnails") + .getObject(0) + .getString("url"); if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; @@ -330,19 +327,24 @@ public boolean isVerified() throws ParsingException { public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonObject videoTabJsonObject = getVideoTab(); Page nextPage = null; - if (getVideoTab() != null) { - final JsonObject gridRenderer = getVideoTab().getObject("content") + if (videoTabJsonObject != null) { + final JsonObject gridRenderer = videoTabJsonObject.getObject("content") .getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("gridRenderer"); + .getArray("contents") + .getObject(0) + .getObject("itemSectionRenderer") + .getArray("contents") + .getObject(0) + .getObject("gridRenderer"); final List channelIds = new ArrayList<>(); channelIds.add(getName()); channelIds.add(getUrl()); - final JsonObject continuation = collectStreamsFrom(collector, gridRenderer - .getArray("items"), channelIds); + final JsonObject continuation = collectStreamsFrom(collector, + gridRenderer.getArray("items"), channelIds); nextPage = getNextPageFrom(continuation, channelIds); } @@ -351,8 +353,8 @@ public InfoItemsPage getInitialPage() throws IOException, Extrac } @Override - public InfoItemsPage getPage(final Page page) throws IOException, - ExtractionException { + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -360,14 +362,10 @@ public InfoItemsPage getPage(final Page page) throws IOException final List channelIds = page.getIds(); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); - final Response response = getDownloader().post(page.getUrl(), null, page.getBody(), + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), getExtractorLocalization()); - final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); - final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") .getObject(0) .getObject("appendContinuationItemsAction"); @@ -380,8 +378,8 @@ public InfoItemsPage getPage(final Page page) throws IOException @Nullable private Page getNextPageFrom(final JsonObject continuations, - final List channelIds) throws IOException, - ExtractionException { + final List channelIds) + throws IOException, ExtractionException { if (isNullOrEmpty(continuations)) { return null; } @@ -394,7 +392,7 @@ private Page getNextPageFrom(final JsonObject continuations, getExtractorContentCountry()) .value("continuation", continuation) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); @@ -444,39 +442,43 @@ public String getUploaderUrl() { @Nullable private JsonObject getVideoTab() throws ParsingException { - if (this.videoTab != null) { - return this.videoTab; + if (videoTab != null) { + return videoTab; } final JsonArray tabs = initialData.getObject("contents") .getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); - JsonObject foundVideoTab = null; - for (final Object tab : tabs) { - if (((JsonObject) tab).has("tabRenderer")) { - if (((JsonObject) tab).getObject("tabRenderer").getString("title", - EMPTY_STRING).equals("Videos")) { - foundVideoTab = ((JsonObject) tab).getObject("tabRenderer"); - break; - } - } - } - - if (foundVideoTab == null) { - throw new ContentNotSupportedException("This channel has no Videos tab"); - } - - final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("messageRenderer").getObject("text")); + final JsonObject foundVideoTab = tabs.stream() + .filter(Objects::nonNull) + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(tab -> tab.has("tabRenderer") + && tab.getObject("tabRenderer") + .getString("title", EMPTY_STRING) + .equals("Videos")) + .findFirst() + .map(tab -> tab.getObject("tabRenderer")) + .orElseThrow( + () -> new ContentNotSupportedException("This channel has no Videos tab")); + + final String messageRendererText = getTextFromObject( + foundVideoTab.getObject("content") + .getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) + .getObject("itemSectionRenderer") + .getArray("contents") + .getObject(0) + .getObject("messageRenderer") + .getObject("text")); if (messageRendererText != null && messageRendererText.equals("This channel has no videos.")) { return null; } - this.videoTab = foundVideoTab; + videoTab = foundVideoTab; return foundVideoTab; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 49225330be..224296d62b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -38,6 +38,8 @@ import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -90,13 +92,16 @@ public void onFetchPage(@Nonnull final Downloader downloader) final Map> headers = new HashMap<>(); addClientInfoHeaders(headers); + headers.put("Content-Type", Collections.singletonList("application/json")); final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization); initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); - playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("playlist").getObject("playlist"); + playlistData = initialData.getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("playlist") + .getObject("playlist"); if (isNullOrEmpty(playlistData)) { throw new ExtractionException("Could not get playlistData"); } @@ -121,8 +126,10 @@ public String getThumbnailUrl() throws ParsingException { } catch (final Exception e) { try { // Fallback to thumbnail of current video. Always the case for channel mix - return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint") - .getObject("watchEndpoint").getString("videoId")); + return getThumbnailUrlFromVideoId( + initialData.getObject("currentVideoEndpoint") + .getObject("watchEndpoint") + .getString("videoId")); } catch (final Exception ignored) { } @@ -211,14 +218,21 @@ public InfoItemsPage getPage(final Page page) throws IOException } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final Map> headers = new HashMap<>(); + headers.put("Content-Type", Collections.singletonList("application/json")); + addClientInfoHeaders(headers); + //noinspection ArraysAsListWithZeroOrOneArgument + headers.put("Cookie", Arrays.asList(page.getCookies().get(COOKIE_NAME))); final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), getExtractorLocalization()); final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonObject playlistJson = ajaxJson.getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); + .getObject("twoColumnWatchNextResults") + .getObject("playlist") + .getObject("playlist"); final JsonArray allStreams = playlistJson.getArray("contents"); // Sublist because YouTube returns up to 24 previous streams in the mix // +1 because the stream of "currentIndex" was already extracted in previous request @@ -247,7 +261,7 @@ private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collecto } @Nonnull - private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) + private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { return getThumbnailUrlFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId)); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index bcbabbcb75..2a3310731f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -5,13 +5,13 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -39,10 +39,9 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -113,18 +112,11 @@ public void onFetchPage(@Nonnull final Downloader downloader) .end() .value("query", getSearchString()) .value("params", params) - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, - json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(url, + getYoutubeMusicHeaders(), json)); try { initialData = JsonParser.object().from(responseBody); @@ -251,18 +243,11 @@ public InfoItemsPage getPage(final Page page) .value("enableSafetyMode", false) .end() .end() - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), - headers, json)); + getYoutubeMusicHeaders(), json)); final JsonObject ajaxJson; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 85c7d8f4ea..310179f774 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -2,14 +2,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -17,11 +15,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; - import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -32,17 +28,12 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; public class YoutubePlaylistExtractor extends PlaylistExtractor { // Minimum size of the stats array in the browse response which includes the streams count @@ -280,12 +271,9 @@ public InfoItemsPage getPage(final Page page) throws IOException } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); - final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), getExtractorLocalization()); - final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions") .getObject(0) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index de937cb021..33e66c56df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -5,21 +5,17 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; - import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -28,17 +24,16 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; -import java.util.HashMap; +import java.nio.charset.StandardCharsets; import java.util.List; -import javax.annotation.Nonnull; - /* * Created by Christian Schabesberger on 22.07.2018 * @@ -90,7 +85,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException jsonBody.value("params", params); } - final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8); + final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8); initialData = getJsonPostResponse("search", body, localization); } @@ -105,10 +100,14 @@ public String getUrl() throws ParsingException { @Override public String getSearchSuggestion() throws ParsingException { final JsonObject itemSectionRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) .getObject("itemSectionRenderer"); - final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0) .getObject("didYouMeanRenderer"); final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents") .getObject(0) @@ -138,8 +137,10 @@ public boolean isCorrectedSearch() { @Override public List getMetaInfo() throws ParsingException { return YoutubeParsingHelper.getMetaInfo( - initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer") + initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") .getArray("contents")); } @@ -149,20 +150,23 @@ public InfoItemsPage getInitialPage() throws IOException, ExtractionEx final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final JsonArray sections = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents"); + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") + .getArray("contents"); Page nextPage = null; for (final Object section : sections) { - if (((JsonObject) section).has("itemSectionRenderer")) { - final JsonObject itemSectionRenderer = ((JsonObject) section) - .getObject("itemSectionRenderer"); + final JsonObject sectionJsonObject = (JsonObject) section; + if (sectionJsonObject.has("itemSectionRenderer")) { + final JsonObject itemSectionRenderer = + sectionJsonObject.getObject("itemSectionRenderer"); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - } else if (((JsonObject) section).has("continuationItemRenderer")) { - nextPage = getNextPageFrom(((JsonObject) section) - .getObject("continuationItemRenderer")); + } else if (sectionJsonObject.has("continuationItemRenderer")) { + nextPage = getNextPageFrom( + sectionJsonObject.getObject("continuationItemRenderer")); } } @@ -184,25 +188,19 @@ public InfoItemsPage getPage(final Page page) throws IOException, getExtractorContentCountry()) .value("continuation", page.getId()) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); // @formatter:on - final String responseBody = getValidJsonResponseBody(getDownloader().post( - page.getUrl(), new HashMap<>(), json)); - - final JsonObject ajaxJson; - try { - ajaxJson = JsonParser.object().from(responseBody); - } catch (final JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } + final JsonObject ajaxJson = getJsonPostResponse("search", json, localization); final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands") - .getObject(0).getObject("appendContinuationItemsAction") + .getObject(0) + .getObject("appendContinuationItemsAction") .getArray("continuationItems"); final JsonArray contents = continuationItems.getObject(0) - .getObject("itemSectionRenderer").getArray("contents"); + .getObject("itemSectionRenderer") + .getArray("contents"); collectStreamsFrom(collector, contents); return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1) @@ -210,28 +208,30 @@ public InfoItemsPage getPage(final Page page) throws IOException, } private void collectStreamsFrom(final MultiInfoItemsCollector collector, - final JsonArray contents) throws NothingFoundException, - ParsingException { + @Nonnull final JsonArray contents) + throws NothingFoundException, ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (final Object content : contents) { final JsonObject item = (JsonObject) content; if (item.has("backgroundPromoRenderer")) { - throw new NothingFoundException(getTextFromObject( - item.getObject("backgroundPromoRenderer").getObject("bodyText"))); + throw new NothingFoundException( + getTextFromObject(item.getObject("backgroundPromoRenderer") + .getObject("bodyText"))); } else if (item.has("videoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(item - .getObject("videoRenderer"), timeAgoParser)); + collector.commit(new YoutubeStreamInfoItemExtractor( + item.getObject("videoRenderer"), timeAgoParser)); } else if (item.has("channelRenderer")) { - collector.commit(new YoutubeChannelInfoItemExtractor(item - .getObject("channelRenderer"))); + collector.commit(new YoutubeChannelInfoItemExtractor( + item.getObject("channelRenderer"))); } else if (item.has("playlistRenderer")) { - collector.commit(new YoutubePlaylistInfoItemExtractor(item - .getObject("playlistRenderer"))); + collector.commit(new YoutubePlaylistInfoItemExtractor( + item.getObject("playlistRenderer"))); } } } + @Nullable private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException { if (isNullOrEmpty(continuationItemRenderer)) { @@ -239,7 +239,8 @@ private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws I } final String token = continuationItemRenderer.getObject("continuationEndpoint") - .getObject("continuationCommand").getString("token"); + .getObject("continuationCommand") + .getString("token"); final String url = YOUTUBEI_V1_URL + "search?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER;