Merge pull request #1518 from CastagnaIT/Leia_newchanges

Partial backports from Matrix

resources/language/resource.language.en_gb/strings.po

@@ -1091,3 +1091,7 @@ msgstr ""
 msgctxt "#30502"
 msgid "MSL manifest version"
 msgstr ""
+
+msgctxt "#30503"
+msgid "Auto generates new ESNs (workaround for 540p limit) not works when Manual ESN is set"
+msgstr ""

resources/lib/common/device_utils.py

@@ -116,7 +116,7 @@ def get_user_agent(enable_android_mediaflag_fix=False):
         # the Windows UA is not limited, so we can use it to get the right video media flags.
         system = 'windows'
 
-    chrome_version = 'Chrome/84.0.4147.136'
+    chrome_version = 'Chrome/108.0.0.0'
     base = 'Mozilla/5.0 '
     base += '%PL% '
     base += 'AppleWebKit/537.36 (KHTML, like Gecko) '
@@ -129,7 +129,7 @@ def get_user_agent(enable_android_mediaflag_fix=False):
     # ARM based Linux
     if get_machine().startswith('arm'):
         # Last number is the platform version of Chrome OS
-        return base.replace('%PL%', '(X11; CrOS armv7l 13099.110.0)')
+        return base.replace('%PL%', '(X11; CrOS armv7l 15183.69.0)')
     # x86 Linux
     return base.replace('%PL%', '(X11; Linux x86_64)')
 

resources/lib/navigation/actions.py

@@ -9,6 +9,8 @@
 """
 from __future__ import absolute_import, division, unicode_literals
 
+import time
+
 import xbmc
 
 import resources.lib.common as common
@@ -184,6 +186,7 @@ def reset_esn(self, pathitems=None):  # pylint: disable=unused-argument
         G.LOCAL_DB.set_value('custom_esn', '', TABLE_SETTINGS_MONITOR)
         # Save the new ESN
         G.LOCAL_DB.set_value('esn', generated_esn, TABLE_SESSION)
+        G.LOCAL_DB.set_value('esn_timestamp', int(time.time()))
         # Reinitialize the MSL handler (delete msl data file, then reset everything)
         common.send_signal(signal=common.Signals.REINITIALIZE_MSL_HANDLER, data=True)
         # Show login notification

resources/lib/services/msl/converter.py

@@ -303,7 +303,11 @@ def _convert_text_track(text_track, period, default, cdn_index, isa_version):
         'Representation',  # Tag
         id=str(list(text_track['downloadableIds'].values())[0]),
         nflxProfile=content_profile)
-    _add_base_url(representation, list(downloadable[content_profile]['downloadUrls'].values())[cdn_index])
+    if 'urls' in downloadable[content_profile]:
+        # The path change when "useBetterTextUrls" param is enabled on manifest
+        _add_base_url(representation, downloadable[content_profile]['urls'][cdn_index]['url'])
+    else:
+        _add_base_url(representation, list(downloadable[content_profile]['downloadUrls'].values())[cdn_index])
 
 
 def _get_id_default_audio_tracks(manifest):

resources/lib/services/msl/msl_handler.py

@@ -21,7 +21,7 @@
 from resources.lib.common.exceptions import CacheMiss, MSLError
 from resources.lib.database.db_utils import TABLE_SESSION
 from resources.lib.globals import G
-from resources.lib.utils.esn import get_esn
+from resources.lib.utils.esn import get_esn, regen_esn
 from resources.lib.utils.logging import LOG, measure_exec_time_decorator
 from .converter import convert_to_dash
 from .events_handler import EventsHandler
@@ -131,7 +131,10 @@ def load_manifest(self, viewable_id):
         :return: MPD XML Manifest or False if no success
         """
         try:
-            manifest = self._load_manifest(viewable_id, get_esn())
+            esn = get_esn()
+            if not G.ADDON.getSetting('esn'):  # Do this only when "Manual ESN" setting is not set
+                esn = regen_esn(esn)
+            manifest = self._load_manifest(viewable_id, esn)
         except MSLError as exc:
             if 'Email or password is incorrect' in G.py2_decode(str(exc)):
                 # Known cases when MSL error "Email or password is incorrect." can happen:
@@ -288,6 +291,9 @@ def _build_manifest_v2(self, **kwargs):
             'uiVersion': G.LOCAL_DB.get_value('ui_version', '', table=TABLE_SESSION),
             'uiPlatform': 'SHAKTI',
             'clientVersion': G.LOCAL_DB.get_value('client_version', '', table=TABLE_SESSION),
+            'platform': G.LOCAL_DB.get_value('browser_info_version', '', table=TABLE_SESSION),
+            'osVersion': G.LOCAL_DB.get_value('browser_info_os_version', '', table=TABLE_SESSION),
+            'osName': G.LOCAL_DB.get_value('browser_info_os_name', '', table=TABLE_SESSION),
             'supportsPreReleasePin': True,
             'supportsWatermark': True,
             'showAllSubDubTracks': False,
@@ -309,7 +315,9 @@ def _build_manifest_v2(self, **kwargs):
             'desiredSegmentVmaf': 'plus_lts',
             'requestSegmentVmaf': False,
             'supportsPartialHydration': False,
-            'contentPlaygraph': [],
+            'contentPlaygraph': ['start'],
+            'liveMetadataFormat': 'INDEXED_SEGMENT_TEMPLATE',
+            'useBetterTextUrls': True,
             'profileGroups': [{
                 'name': 'default',
                 'profiles': kwargs['profiles']

resources/lib/utils/esn.py

@@ -9,7 +9,8 @@
 """
 from __future__ import absolute_import, division, unicode_literals
 
-from re import sub
+import re
+import time
 
 from resources.lib.database.db_utils import TABLE_SESSION
 from resources.lib.globals import G
@@ -55,6 +56,36 @@ def get_esn():
     return custom_esn if custom_esn else G.LOCAL_DB.get_value('esn', '', table=TABLE_SESSION)
 
 
+def regen_esn(esn):
+    # From the beginning of December 2022 if you are using an ESN for more than about 20 hours
+    # Netflix limits the resolution to 540p. The reasons behind this are unknown, there are no changes on website
+    # or Android apps. Moreover, if you set the full-length ESN of android app on the add-on, also the original app
+    # will be downgraded to 540p without any kind of message.
+    if not G.ADDON.getSettingBool('esn_auto_generate'):
+        return esn
+    from resources.lib.common.device_utils import get_system_platform
+    ts_now = int(time.time())
+    ts_esn = G.LOCAL_DB.get_value('esn_timestamp', default_value=0)
+    # When an ESN has been used for more than 20 hours ago, generate a new ESN
+    if ts_esn == 0 or ts_now - ts_esn > 72000:
+        if get_system_platform() == 'android':
+            if esn[-1] == '-':
+                # We have a partial ESN without last 64 chars, so generate and add the 64 chars
+                esn += _create_id64chars()
+            elif re.search(r'-[0-9]+-[A-Z0-9]{64}', esn):
+                # Replace last 64 chars with the new generated one
+                esn = esn[:-64] + _create_id64chars()
+            else:
+                LOG.warn('ESN format not recognized, will be reset with a new ESN')
+                esn = generate_android_esn()
+        else:
+            esn = generate_esn(esn[:-30])
+        G.LOCAL_DB.set_value('esn', esn, table=TABLE_SESSION)
+        G.LOCAL_DB.set_value('esn_timestamp', ts_now)
+        LOG.debug('The ESN has been regenerated (540p workaround).')
+    return esn
+
+
 def generate_android_esn():
     """Generate an ESN if on android or return the one from user_data"""
     from resources.lib.common.device_utils import get_system_platform
@@ -101,23 +132,38 @@ def generate_android_esn():
 
                 esn += '{:=<5.5}'.format(manufacturer)
                 esn += model[:45].replace(' ', '=')
-                esn = sub(r'[^A-Za-z0-9=-]', '=', esn)
-                if system_id:
-                    esn += '-' + system_id + '-'
+                esn = re.sub(r'[^A-Za-z0-9=-]', '=', esn)
+                esn += '-' + system_id + '-' + _create_id64chars()
                 LOG.debug('Generated Android ESN: {} (force widevine is set as "{}")', esn, force_widevine)
                 return esn
         except OSError:
             pass
     return None
 
 
-def generate_esn(prefix=''):
-    """Generate a random ESN"""
-    # For possibles prefixes see website, are based on browser user agent
-    import random
-    esn = prefix
+def generate_esn(init_part=None):
+    """
+    Generate a random ESN
+    :param init_part: Specify the initial part to be used e.g. "NFCDCH-02-",
+                      if not set will be obtained from the last retrieved from the website
+    :return: The generated ESN
+    """
+    # The initial part of the ESN e.g. "NFCDCH-02-" depends on the web browser used and then the user agent,
+    # refer to website to know all types available.
+    if not init_part:
+        raise Exception('Cannot generate ESN due to missing initial ESN part')
+    esn = init_part
     possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+    from random import choice
     for _ in range(0, 30):
-        esn += random.choice(possible)
-    LOG.debug('Generated random ESN: {}', esn)
+        esn += choice(possible)
     return esn
+
+
+def _create_id64chars():
+    # The Android full length ESN include to the end a hashed ID of 64 chars,
+    # this value is created from the android app by using the Widevine "deviceUniqueId" property value
+    # hashed in various ways, not knowing the correct formula, we create a random value.
+    # Starting from 12/2022 this value is mandatory to obtain HD resolutions
+    from os import urandom
+    return re.sub(r'[^A-Za-z0-9=-]', '=', urandom(32).encode('hex').upper())

resources/lib/utils/website.py

@@ -10,6 +10,7 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import json
+import time
 from re import search, compile as recompile, DOTALL, sub
 
 from future.utils import iteritems, raise_from
@@ -55,9 +56,9 @@
     'request_id': 'models/serverDefs/data/requestId',
     'asset_core': 'models/playerModel/data/config/core/assets/core',
     'ui_version': 'models/playerModel/data/config/ui/initParams/uiVersion',
-    'browser_info_version': 'models/browserInfo/data/version',
-    'browser_info_os_name': 'models/browserInfo/data/os/name',
-    'browser_info_os_version': 'models/browserInfo/data/os/version',
+    'browser_info_version': 'models/playerModel/data/config/core/initParams/browserInfo/version',
+    'browser_info_os_name': 'models/playerModel/data/config/core/initParams/browserInfo/os/name',
+    'browser_info_os_version': 'models/playerModel/data/config/core/initParams/browserInfo/os/version',
 }
 
 PAGE_ITEM_ERROR_CODE = 'models/flow/data/fields/errorCode/value'
@@ -92,6 +93,7 @@ def extract_session_data(content, validate=False, update_profiles=False):
     # Save only some info of the current profile from user data
     G.LOCAL_DB.set_value('build_identifier', user_data.get('BUILD_IDENTIFIER'), TABLE_SESSION)
     if not G.LOCAL_DB.get_value('esn', table=TABLE_SESSION):
+        G.LOCAL_DB.set_value('esn_timestamp', int(time.time()))
         G.LOCAL_DB.set_value('esn', generate_android_esn() or user_data['esn'], TABLE_SESSION)
     G.LOCAL_DB.set_value('locale_id', user_data.get('preferredLocale').get('id', 'en-US'))
     # Extract the client version from assets core
@@ -286,6 +288,7 @@ def extract_json(content, name):
         json_str_replace = json_str_replace.replace(r'\r', r'\\r')  # Escape return
         json_str_replace = json_str_replace.replace(r'\n', r'\\n')  # Escape line feed
         json_str_replace = json_str_replace.replace(r'\t', r'\\t')  # Escape tab
+        json_str_replace = json_str_replace.replace(r'\p', r'/p')  # Unicode property not supported, we change slash to avoid unescape it
         json_str_replace = json_str_replace.encode().decode('unicode_escape')  # Decode the string as unicode
         json_str_replace = sub(r'\\(?!["])', r'\\\\', json_str_replace)  # Escape backslash (only when is not followed by double quotation marks \")
         return json.loads(json_str_replace)

resources/settings.xml

@@ -145,6 +145,7 @@
     <setting id="view_esn" type="action" label="30216" action="RunPlugin(plugin://$ID/action/view_esn/)"/>
     <setting id="esn" type="text" label="30034" value="" default=""/>
     <setting id="reset_esn" type="action" label="30217" action="RunPlugin(plugin://$ID/action/reset_esn/)" option="close"/>
+    <setting id="esn_auto_generate" type="bool" label="30503" default="true"/>
     <setting label="30117" type="lsep"/><!--Cache-->
     <setting id="cache_ttl" type="slider" option="int" range="5,5,2880" label="30084" default="180"/>
     <setting id="cache_mylist_ttl" type="slider" option="int" range="10,10,1440" label="30086" default="60"/>