Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YouTube] Update clients version and improve extraction of API key and client version of WEB client #890

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -932,6 +932,10 @@ private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentC
mobileBody, localization, "&t=" + generateTParameter()
+ "&id=" + videoId);

if (isPlayerResponseNotValid(androidPlayerResponse, videoId)) {
return;
}

final JsonObject streamingData = androidPlayerResponse.getObject(STREAMING_DATA);
if (!isNullOrEmpty(streamingData)) {
androidStreamingData = streamingData;
Expand Down Expand Up @@ -963,6 +967,10 @@ private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCount
mobileBody, localization, "&t=" + generateTParameter()
+ "&id=" + videoId);

if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) {
return;
}

final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA);
if (!isNullOrEmpty(streamingData)) {
iosStreamingData = streamingData;
Expand Down Expand Up @@ -1001,6 +1009,38 @@ private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCo
}
}

/**
* Checks whether an additional player response is not valid.
*
* <p>
* If YouTube detect that requests come from a third party client, they may replace the real
* player response by another one of a video saying that this content is not available on this
* app and to watch it on the latest version of YouTube.
* </p>
*
* <p>
* We can detect this by checking whether the video ID of the player response returned is the
* same as the one requested by the extractor.
* </p>
*
* <p>
* This behavior has been already observed on the {@code ANDROID} client, see
* <a href="https://github.com/TeamNewPipe/NewPipe/issues/8713">
* https://github.com/TeamNewPipe/NewPipe/issues/8713</a>.
* </p>
*
* @param additionalPlayerResponse an additional response to the one of the {@code HTML5}
* client used
* @param videoId the video ID of the content requested
* @return whether the video ID of the player response is not equal to the one requested
*/
private static boolean isPlayerResponseNotValid(
@Nonnull final JsonObject additionalPlayerResponse,
@Nonnull final String videoId) {
return !videoId.equals(additionalPlayerResponse.getObject("videoDetails")
.getString("videoId", ""));
}

private static void storePlayerJs() throws ParsingException {
try {
playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
],
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
Expand All @@ -37,14 +37,17 @@
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"critical-ch": [
"Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Model, Sec-CH-UA-Platform, Sec-CH-UA-Platform-Version"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\""
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Tue, 15 Mar 2022 17:06:48 GMT"
"Fri, 12 Aug 2022 17:15:42 GMT"
],
"expires": [
"Tue, 15 Mar 2022 17:06:48 GMT"
"Fri, 12 Aug 2022 17:15:42 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
Expand All @@ -53,15 +56,15 @@
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/encsid_ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\"}]}"
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dk7SuAYyBwyE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 19-Jun-2019 17:06:48 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+647; expires\u003dThu, 14-Mar-2024 17:06:48 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003dzvoZRVLWdUQ; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 16-Nov-2019 17:15:42 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+621; expires\u003dSun, 11-Aug-2024 17:15:42 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
Expand All @@ -76,7 +79,7 @@
"0"
]
},
"responseBody": "/** 18082585275338869083 */self.document \u003d self; self.window \u003d self;var ytcfg\u003d{d:function(){return window.yt\u0026\u0026yt.config_||ytcfg.data_||(ytcfg.data_\u003d{})},get:function(k,o){return k in ytcfg.d()?ytcfg.d()[k]:o},set:function(){var a\u003darguments;if(a.length\u003e1)ytcfg.d()[a[0]]\u003da[1];else for(var k in a[0])ytcfg.d()[k]\u003da[0][k]}};\nytcfg.set({\"EXPERIMENT_FLAGS\":{\"allow_music_base_url\":true,\"allow_skip_networkless\":true,\"clear_user_partitioned_ls\":true,\"deprecate_two_way_binding_child\":true,\"deprecate_two_way_binding_parent\":true,\"desktop_image_cta_no_background\":true,\"desktop_text_ads_gray_visurl\":true,\"disable_child_node_auto_formatted_strings\":true,\"disable_simple_mixed_direction_formatted_strings\":true,\"disable_thumbnail_preloading\":true,\"enable_client_sli_logging\":true,\"enable_gel_log_commands\":true,\"enable_gray_visurl\":true,\"enable_mixed_direction_formatted_strings\":true,\"enable_server_stitched_dai\":true,\"enable_sli_flush\":true,\"enable_tectonic_ad_ux_for_halftime\":true,\"enable_topsoil_wta_for_halftime_live_infra\":true,\"enable_web_eom_visitor_data\":true,\"export_networkless_options\":true,\"forward_domain_admin_state_on_embeds\":true,\"html5_enable_single_video_vod_ivar_on_pacf\":true,\"html5_enable_video_overlay_on_inplayer_slot_for_tv\":true,\"html5_pacf_enable_dai\":true,\"html5_recognize_predict_start_cue_point\":true,\"html5_server_stitched_dai_group\":true,\"json_condensed_response\":true,\"kevlar_dropdown_fix\":true,\"kevlar_gel_error_routing\":true,\"log_heartbeat_with_lifecycles\":true,\"log_web_endpoint_to_layer\":true,\"networkless_gel\":true,\"networkless_logging\":true,\"nwl_send_fast_on_unload\":true,\"offline_error_handling\":true,\"pageid_as_header_web\":true,\"pes_migrate_association_data\":true,\"polymer_bad_build_labels\":true,\"polymer_verifiy_app_state\":true,\"qoe_send_and_write\":true,\"record_app_crashed_web\":true,\"search_image_menu_alignment\":true,\"skip_ls_gel_retry\":true,\"suppress_error_204_logging\":true,\"sw_nav_request_network_first\":true,\"use_bg_facade\":true,\"use_document_lifecycles\":true,\"use_screen_manager_util\":true,\"use_undefined_csn_any_layer\":true,\"vss_final_ping_send_and_write\":true,\"vss_playback_use_send_and_write\":true,\"web_api_url\":true,\"web_click_command_fallback\":true,\"web_dedupe_ve_grafting\":true,\"web_deprecate_service_ajax_map_dependency\":true,\"web_enable_ad_signals_in_it_context\":true,\"web_fetch_promise_cleanup_killswitch\":true,\"web_forward_command_on_pbj\":true,\"web_log_memory_total_kbytes\":true,\"web_ordered_response_processors\":true,\"web_prefetch_preload_video\":true,\"web_yt_config_context\":true,\"ytidb_fetch_datasync_ids_for_data_cleanup\":true,\"addto_ajax_log_warning_fraction\":0.1,\"log_window_onerror_fraction\":0.1,\"ytidb_transaction_ended_event_rate_limit\":0.02,\"botguard_async_snapshot_timeout_ms\":3000,\"check_navigator_accuracy_timeout_ms\":0,\"initial_gel_batch_timeout\":2000,\"network_polling_interval\":30000,\"polymer_log_prop_change_observer_percent\":0,\"web_foreground_heartbeat_interval_ms\":28000,\"web_logging_max_batch\":150,\"ytidb_remake_db_retries\":1,\"ytidb_reopen_db_retries\":0,\"cb_v2_uxe\":\"23983171\",\"web_client_version_override\":\"\",\"kevlar_command_handler_command_banlist\":[],\"web_op_continuation_type_banlist\":[],\"web_op_endpoint_banlist\":[],\"web_op_signal_type_banlist\":[]},\"INNERTUBE_API_KEY\":\"AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\",\"INNERTUBE_API_VERSION\":\"v1\",\"INNERTUBE_CLIENT_NAME\":\"WEB\",\"INNERTUBE_CLIENT_VERSION\":\"2.20220315.01.00\",\"INNERTUBE_CONTEXT\":{\"client\":{\"clientName\":\"WEB\",\"clientVersion\":\"2.20220315.01.00\"}},\"INNERTUBE_CONTEXT_CLIENT_NAME\":1,\"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"2.20220315.01.00\",\"LATEST_ECATCHER_SERVICE_TRACKING_PARAMS\":{\"client.name\":\"WEB\"}});window.ytcfg.obfuscatedData_ \u003d [];if (self.trustedTypes \u0026\u0026 self.trustedTypes.createPolicy) {const swPolicy \u003d trustedTypes.createPolicy(\u0027youtubeServiceWorkerPolicy\u0027, {createScriptURL: function(ignored) { return \u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/d3f3a825\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027; }}); importScripts(swPolicy.createScriptURL(\u0027\u0027));} else {importScripts(\u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/d3f3a825\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027);}",
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}
Loading