diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po index 8ff6d93e8..b34ea0009 100644 --- a/resources/language/resource.language.en_gb/strings.po +++ b/resources/language/resource.language.en_gb/strings.po @@ -1091,3 +1091,7 @@ msgstr "" msgctxt "#30502" msgid "MSL manifest version" msgstr "" + +msgctxt "#30503" +msgid "Auto generates new ESNs (workaround for 540p limit) not works when Manual ESN is set" +msgstr "" diff --git a/resources/lib/common/device_utils.py b/resources/lib/common/device_utils.py index 5f9b20602..5172c20c9 100644 --- a/resources/lib/common/device_utils.py +++ b/resources/lib/common/device_utils.py @@ -116,7 +116,7 @@ def get_user_agent(enable_android_mediaflag_fix=False): # the Windows UA is not limited, so we can use it to get the right video media flags. system = 'windows' - chrome_version = 'Chrome/84.0.4147.136' + chrome_version = 'Chrome/108.0.0.0' base = 'Mozilla/5.0 ' base += '%PL% ' base += 'AppleWebKit/537.36 (KHTML, like Gecko) ' @@ -129,7 +129,7 @@ def get_user_agent(enable_android_mediaflag_fix=False): # ARM based Linux if get_machine().startswith('arm'): # Last number is the platform version of Chrome OS - return base.replace('%PL%', '(X11; CrOS armv7l 13099.110.0)') + return base.replace('%PL%', '(X11; CrOS armv7l 15183.69.0)') # x86 Linux return base.replace('%PL%', '(X11; Linux x86_64)') diff --git a/resources/lib/navigation/actions.py b/resources/lib/navigation/actions.py index bf1bd243e..0267c2f85 100644 --- a/resources/lib/navigation/actions.py +++ b/resources/lib/navigation/actions.py @@ -9,6 +9,8 @@ """ from __future__ import absolute_import, division, unicode_literals +import time + import xbmc import resources.lib.common as common @@ -184,6 +186,7 @@ def reset_esn(self, pathitems=None): # pylint: disable=unused-argument G.LOCAL_DB.set_value('custom_esn', '', TABLE_SETTINGS_MONITOR) # Save the new ESN G.LOCAL_DB.set_value('esn', generated_esn, TABLE_SESSION) + G.LOCAL_DB.set_value('esn_timestamp', int(time.time())) # Reinitialize the MSL handler (delete msl data file, then reset everything) common.send_signal(signal=common.Signals.REINITIALIZE_MSL_HANDLER, data=True) # Show login notification diff --git a/resources/lib/services/msl/converter.py b/resources/lib/services/msl/converter.py index 8d5fb4c36..4ea9b2c6f 100644 --- a/resources/lib/services/msl/converter.py +++ b/resources/lib/services/msl/converter.py @@ -303,7 +303,11 @@ def _convert_text_track(text_track, period, default, cdn_index, isa_version): 'Representation', # Tag id=str(list(text_track['downloadableIds'].values())[0]), nflxProfile=content_profile) - _add_base_url(representation, list(downloadable[content_profile]['downloadUrls'].values())[cdn_index]) + if 'urls' in downloadable[content_profile]: + # The path change when "useBetterTextUrls" param is enabled on manifest + _add_base_url(representation, downloadable[content_profile]['urls'][cdn_index]['url']) + else: + _add_base_url(representation, list(downloadable[content_profile]['downloadUrls'].values())[cdn_index]) def _get_id_default_audio_tracks(manifest): diff --git a/resources/lib/services/msl/msl_handler.py b/resources/lib/services/msl/msl_handler.py index 34fce8be6..679b67b7f 100644 --- a/resources/lib/services/msl/msl_handler.py +++ b/resources/lib/services/msl/msl_handler.py @@ -21,7 +21,7 @@ from resources.lib.common.exceptions import CacheMiss, MSLError from resources.lib.database.db_utils import TABLE_SESSION from resources.lib.globals import G -from resources.lib.utils.esn import get_esn +from resources.lib.utils.esn import get_esn, regen_esn from resources.lib.utils.logging import LOG, measure_exec_time_decorator from .converter import convert_to_dash from .events_handler import EventsHandler @@ -131,7 +131,10 @@ def load_manifest(self, viewable_id): :return: MPD XML Manifest or False if no success """ try: - manifest = self._load_manifest(viewable_id, get_esn()) + esn = get_esn() + if not G.ADDON.getSetting('esn'): # Do this only when "Manual ESN" setting is not set + esn = regen_esn(esn) + manifest = self._load_manifest(viewable_id, esn) except MSLError as exc: if 'Email or password is incorrect' in G.py2_decode(str(exc)): # Known cases when MSL error "Email or password is incorrect." can happen: @@ -288,6 +291,9 @@ def _build_manifest_v2(self, **kwargs): 'uiVersion': G.LOCAL_DB.get_value('ui_version', '', table=TABLE_SESSION), 'uiPlatform': 'SHAKTI', 'clientVersion': G.LOCAL_DB.get_value('client_version', '', table=TABLE_SESSION), + 'platform': G.LOCAL_DB.get_value('browser_info_version', '', table=TABLE_SESSION), + 'osVersion': G.LOCAL_DB.get_value('browser_info_os_version', '', table=TABLE_SESSION), + 'osName': G.LOCAL_DB.get_value('browser_info_os_name', '', table=TABLE_SESSION), 'supportsPreReleasePin': True, 'supportsWatermark': True, 'showAllSubDubTracks': False, @@ -309,7 +315,9 @@ def _build_manifest_v2(self, **kwargs): 'desiredSegmentVmaf': 'plus_lts', 'requestSegmentVmaf': False, 'supportsPartialHydration': False, - 'contentPlaygraph': [], + 'contentPlaygraph': ['start'], + 'liveMetadataFormat': 'INDEXED_SEGMENT_TEMPLATE', + 'useBetterTextUrls': True, 'profileGroups': [{ 'name': 'default', 'profiles': kwargs['profiles'] diff --git a/resources/lib/utils/esn.py b/resources/lib/utils/esn.py index 1446822a5..0ac04103e 100644 --- a/resources/lib/utils/esn.py +++ b/resources/lib/utils/esn.py @@ -9,7 +9,8 @@ """ from __future__ import absolute_import, division, unicode_literals -from re import sub +import re +import time from resources.lib.database.db_utils import TABLE_SESSION from resources.lib.globals import G @@ -55,6 +56,36 @@ def get_esn(): return custom_esn if custom_esn else G.LOCAL_DB.get_value('esn', '', table=TABLE_SESSION) +def regen_esn(esn): + # From the beginning of December 2022 if you are using an ESN for more than about 20 hours + # Netflix limits the resolution to 540p. The reasons behind this are unknown, there are no changes on website + # or Android apps. Moreover, if you set the full-length ESN of android app on the add-on, also the original app + # will be downgraded to 540p without any kind of message. + if not G.ADDON.getSettingBool('esn_auto_generate'): + return esn + from resources.lib.common.device_utils import get_system_platform + ts_now = int(time.time()) + ts_esn = G.LOCAL_DB.get_value('esn_timestamp', default_value=0) + # When an ESN has been used for more than 20 hours ago, generate a new ESN + if ts_esn == 0 or ts_now - ts_esn > 72000: + if get_system_platform() == 'android': + if esn[-1] == '-': + # We have a partial ESN without last 64 chars, so generate and add the 64 chars + esn += _create_id64chars() + elif re.search(r'-[0-9]+-[A-Z0-9]{64}', esn): + # Replace last 64 chars with the new generated one + esn = esn[:-64] + _create_id64chars() + else: + LOG.warn('ESN format not recognized, will be reset with a new ESN') + esn = generate_android_esn() + else: + esn = generate_esn(esn[:-30]) + G.LOCAL_DB.set_value('esn', esn, table=TABLE_SESSION) + G.LOCAL_DB.set_value('esn_timestamp', ts_now) + LOG.debug('The ESN has been regenerated (540p workaround).') + return esn + + def generate_android_esn(): """Generate an ESN if on android or return the one from user_data""" from resources.lib.common.device_utils import get_system_platform @@ -101,9 +132,8 @@ def generate_android_esn(): esn += '{:=<5.5}'.format(manufacturer) esn += model[:45].replace(' ', '=') - esn = sub(r'[^A-Za-z0-9=-]', '=', esn) - if system_id: - esn += '-' + system_id + '-' + esn = re.sub(r'[^A-Za-z0-9=-]', '=', esn) + esn += '-' + system_id + '-' + _create_id64chars() LOG.debug('Generated Android ESN: {} (force widevine is set as "{}")', esn, force_widevine) return esn except OSError: @@ -111,13 +141,29 @@ def generate_android_esn(): return None -def generate_esn(prefix=''): - """Generate a random ESN""" - # For possibles prefixes see website, are based on browser user agent - import random - esn = prefix +def generate_esn(init_part=None): + """ + Generate a random ESN + :param init_part: Specify the initial part to be used e.g. "NFCDCH-02-", + if not set will be obtained from the last retrieved from the website + :return: The generated ESN + """ + # The initial part of the ESN e.g. "NFCDCH-02-" depends on the web browser used and then the user agent, + # refer to website to know all types available. + if not init_part: + raise Exception('Cannot generate ESN due to missing initial ESN part') + esn = init_part possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + from random import choice for _ in range(0, 30): - esn += random.choice(possible) - LOG.debug('Generated random ESN: {}', esn) + esn += choice(possible) return esn + + +def _create_id64chars(): + # The Android full length ESN include to the end a hashed ID of 64 chars, + # this value is created from the android app by using the Widevine "deviceUniqueId" property value + # hashed in various ways, not knowing the correct formula, we create a random value. + # Starting from 12/2022 this value is mandatory to obtain HD resolutions + from os import urandom + return re.sub(r'[^A-Za-z0-9=-]', '=', urandom(32).encode('hex').upper()) diff --git a/resources/lib/utils/website.py b/resources/lib/utils/website.py index 71e31cb31..e00a69cfa 100644 --- a/resources/lib/utils/website.py +++ b/resources/lib/utils/website.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, division, unicode_literals import json +import time from re import search, compile as recompile, DOTALL, sub from future.utils import iteritems, raise_from @@ -55,9 +56,9 @@ 'request_id': 'models/serverDefs/data/requestId', 'asset_core': 'models/playerModel/data/config/core/assets/core', 'ui_version': 'models/playerModel/data/config/ui/initParams/uiVersion', - 'browser_info_version': 'models/browserInfo/data/version', - 'browser_info_os_name': 'models/browserInfo/data/os/name', - 'browser_info_os_version': 'models/browserInfo/data/os/version', + 'browser_info_version': 'models/playerModel/data/config/core/initParams/browserInfo/version', + 'browser_info_os_name': 'models/playerModel/data/config/core/initParams/browserInfo/os/name', + 'browser_info_os_version': 'models/playerModel/data/config/core/initParams/browserInfo/os/version', } PAGE_ITEM_ERROR_CODE = 'models/flow/data/fields/errorCode/value' @@ -92,6 +93,7 @@ def extract_session_data(content, validate=False, update_profiles=False): # Save only some info of the current profile from user data G.LOCAL_DB.set_value('build_identifier', user_data.get('BUILD_IDENTIFIER'), TABLE_SESSION) if not G.LOCAL_DB.get_value('esn', table=TABLE_SESSION): + G.LOCAL_DB.set_value('esn_timestamp', int(time.time())) G.LOCAL_DB.set_value('esn', generate_android_esn() or user_data['esn'], TABLE_SESSION) G.LOCAL_DB.set_value('locale_id', user_data.get('preferredLocale').get('id', 'en-US')) # Extract the client version from assets core @@ -286,6 +288,7 @@ def extract_json(content, name): json_str_replace = json_str_replace.replace(r'\r', r'\\r') # Escape return json_str_replace = json_str_replace.replace(r'\n', r'\\n') # Escape line feed json_str_replace = json_str_replace.replace(r'\t', r'\\t') # Escape tab + json_str_replace = json_str_replace.replace(r'\p', r'/p') # Unicode property not supported, we change slash to avoid unescape it json_str_replace = json_str_replace.encode().decode('unicode_escape') # Decode the string as unicode json_str_replace = sub(r'\\(?!["])', r'\\\\', json_str_replace) # Escape backslash (only when is not followed by double quotation marks \") return json.loads(json_str_replace) diff --git a/resources/settings.xml b/resources/settings.xml index ca4125a92..309c6215e 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -145,6 +145,7 @@ +