From 35cd07d9a4d1f8eef8f91b5a1efe67aee93e7a65 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Sat, 19 Aug 2023 12:01:14 -0700 Subject: [PATCH 01/11] Update Test to not end when format is not correct. --- conformance.py | 2 +- dab/app_telemetry.py | 12 ++++++++++-- dab/applications.py | 29 +++++++++++++++++++++++++---- dab/device.py | 6 +++++- dab/device_telemetry.py | 12 ++++++++++-- dab/health_check.py | 6 +++++- dab/input.py | 18 +++++++++++++++--- dab/operations.py | 6 +++++- dab/output.py | 6 +++++- dab/system.py | 24 ++++++++++++++++++++---- dab/version.py | 6 +++++- dab/voice.py | 24 ++++++++++++++++++++---- 12 files changed, 126 insertions(+), 25 deletions(-) diff --git a/conformance.py b/conformance.py index 33ee83f..4da4bd8 100644 --- a/conformance.py +++ b/conformance.py @@ -16,7 +16,7 @@ ("operations/list",'{}', dab.operations.list, 200, "Conformance"), ("applications/list",'{}', dab.applications.list, 200, "Conformance"), ("applications/launch",f'{{"appId": "{config.apps["youtube"]}"}}', dab.applications.launch, 10000, "Conformance"), - ("applications/launch",f'{{"appId": "{config.apps["youtube"]}", "parameters": "#/watch?list=OLAK5uy_mKAu6VNK3gMSq_L8fU_C6myQnuuuIzvWY"}}', dab.applications.launch, 10000, "with parameters"), + ("applications/launch",f'{{"appId": "{config.apps["youtube"]}", "parameters": ["#/watch?list=OLAK5uy_mKAu6VNK3gMSq_L8fU_C6myQnuuuIzvWY"]}}', dab.applications.launch, 10000, "with parameters"), ("applications/launch-with-content",f'{{"appId": "{config.apps["youtube"]}", "contentId": "jfKfPfyJRdk"}}', dab.applications.launch_with_content, 10000, "Conformance"), ("applications/get-state",f'{{"appId": "{config.apps["youtube"]}"}}', dab.applications.get_state, 200, "Conformance"), ("applications/exit",f'{{"appId": "{config.apps["youtube"]}"}}', dab.applications.exit, 5000, "Conformance"), diff --git a/dab/app_telemetry.py b/dab/app_telemetry.py index e50ae31..5003316 100644 --- a/dab/app_telemetry.py +++ b/dab/app_telemetry.py @@ -5,7 +5,11 @@ def start(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response) + try: + dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -13,6 +17,10 @@ def start(test_result, durationInMs=0,expectedLatencyMs=0): return Default_Validations(test_result, durationInMs, expectedLatencyMs) def stop(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response) + try: + dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) \ No newline at end of file diff --git a/dab/applications.py b/dab/applications.py index 29821e1..11936b1 100644 --- a/dab/applications.py +++ b/dab/applications.py @@ -4,7 +4,11 @@ import jsons def launch(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_dab_response_schema(test_result.response) + try: + dab_response_validator.validate_dab_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -13,6 +17,11 @@ def launch(test_result, durationInMs=0,expectedLatencyMs=0): def launch_with_content(test_result, durationInMs=0,expectedLatencyMs=0): dab_response_validator.validate_dab_response_schema(test_result.response) + try: + dab_response_validator.validate_dab_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -20,7 +29,11 @@ def launch_with_content(test_result, durationInMs=0,expectedLatencyMs=0): return YesNoQuestion(test_result, "Cobalt started with playback?") and Default_Validations(test_result, durationInMs, expectedLatencyMs) def exit(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_exit_application_response_schema(test_result.response) + try: + dab_response_validator.validate_exit_application_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -28,7 +41,11 @@ def exit(test_result, durationInMs=0,expectedLatencyMs=0): return YesNoQuestion(test_result, "Cobalt exited?") and Default_Validations(test_result, durationInMs, expectedLatencyMs) def list(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_list_applications_response_schema(test_result.response) + try: + dab_response_validator.validate_list_applications_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -36,7 +53,11 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): return Default_Validations(test_result, durationInMs, expectedLatencyMs) def get_state(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_get_application_state_response_schema(test_result.response) + try: + dab_response_validator.validate_get_application_state_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/device.py b/dab/device.py index a0caab0..036c87d 100644 --- a/dab/device.py +++ b/dab/device.py @@ -4,7 +4,11 @@ import jsons def info(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_device_information_schema(test_result.response) + try: + dab_response_validator.validate_device_information_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/device_telemetry.py b/dab/device_telemetry.py index 64256c4..d8f01b0 100644 --- a/dab/device_telemetry.py +++ b/dab/device_telemetry.py @@ -4,7 +4,11 @@ import jsons def start(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response) + try: + dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -12,7 +16,11 @@ def start(test_result, durationInMs=0,expectedLatencyMs=0): return Default_Validations(test_result, durationInMs, expectedLatencyMs) def stop(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response) + try: + dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/health_check.py b/dab/health_check.py index 2995e9e..46acdd4 100644 --- a/dab/health_check.py +++ b/dab/health_check.py @@ -4,7 +4,11 @@ import jsons def get(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_health_check_response_schema(test_result.response) + try: + dab_response_validator.validate_health_check_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/input.py b/dab/input.py index 9a31f40..6355a0c 100644 --- a/dab/input.py +++ b/dab/input.py @@ -7,7 +7,11 @@ class KeyList: key_list = [] def key_press(test_result, durationInMs=0, expectedLatencyMs=None): - dab_response_validator.validate_dab_response_schema(test_result.response) + try: + dab_response_validator.validate_dab_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False request = jsons.loads(test_result.request) response = jsons.loads(test_result.response) # No list available, assuming everything is required. @@ -28,7 +32,11 @@ def key_press(test_result, durationInMs=0, expectedLatencyMs=None): return YesNoQuestion(test_result, expectedLatencyMs) def long_key_press(test_result, durationInMs=0, expectedLatencyMs=None): - dab_response_validator.validate_dab_response_schema(test_result.response) + try: + dab_response_validator.validate_dab_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False request = jsons.loads(test_result.request) response = jsons.loads(test_result.response) # No list available, assuming everything is required. @@ -50,7 +58,11 @@ def long_key_press(test_result, durationInMs=0, expectedLatencyMs=None): return YesNoQuestion(test_result, expectedLatencyMs) def list(test_result, durationInMs=0, expectedLatencyMs=0): - dab_response_validator.validate_key_list_schema(test_result.response) + try: + dab_response_validator.validate_key_list_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/operations.py b/dab/operations.py index 4e052c4..191549b 100644 --- a/dab/operations.py +++ b/dab/operations.py @@ -4,7 +4,11 @@ import jsons def list(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_list_supported_operation_response_schema(test_result.response) + try: + dab_response_validator.validate_list_supported_operation_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/output.py b/dab/output.py index 11d169e..1d4b96e 100644 --- a/dab/output.py +++ b/dab/output.py @@ -4,7 +4,11 @@ import jsons def image(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_output_image_response_schema(test_result.response) + try: + dab_response_validator.validate_output_image_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/system.py b/dab/system.py index b6dab40..8f45845 100644 --- a/dab/system.py +++ b/dab/system.py @@ -4,7 +4,11 @@ from schema import dab_response_validator def restart(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_dab_response_schema(test_result.response) + try: + dab_response_validator.validate_dab_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -13,7 +17,11 @@ def restart(test_result, durationInMs=0,expectedLatencyMs=0): return YesNoQuestion(test_result, "Cobalt re-started?") def get(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_get_system_settings_response_schema(test_result.response) + try: + dab_response_validator.validate_get_system_settings_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -21,7 +29,11 @@ def get(test_result, durationInMs=0,expectedLatencyMs=0): return Default_Validations(test_result, durationInMs, expectedLatencyMs) def set(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_set_system_settings_response_schema(test_result.response) + try: + dab_response_validator.validate_set_system_settings_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -29,7 +41,11 @@ def set(test_result, durationInMs=0,expectedLatencyMs=0): return Default_Validations(test_result, durationInMs, expectedLatencyMs) def list(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_list_system_settings_schema(test_result.response) + try: + dab_response_validator.validate_list_system_settings_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/version.py b/dab/version.py index 3402eda..3c9b2d8 100644 --- a/dab/version.py +++ b/dab/version.py @@ -4,7 +4,11 @@ import jsons def default(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_version_response_schema(test_result.response) + try: + dab_response_validator.validate_version_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False diff --git a/dab/voice.py b/dab/voice.py index 9331bc2..99d76a2 100644 --- a/dab/voice.py +++ b/dab/voice.py @@ -4,7 +4,11 @@ import jsons def send_audio(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_dab_response_schema(test_result.response) + try: + dab_response_validator.validate_dab_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -15,7 +19,11 @@ def send_audio(test_result, durationInMs=0,expectedLatencyMs=0): return YesNoQuestion(test_result, expectedLatencyMs) def send_text(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_dab_response_schema(test_result.response) + try: + dab_response_validator.validate_dab_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -26,7 +34,11 @@ def send_text(test_result, durationInMs=0,expectedLatencyMs=0): return YesNoQuestion(test_result, expectedLatencyMs) def list(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_list_voice_response_schema(test_result.response) + try: + dab_response_validator.validate_list_voice_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False @@ -34,7 +46,11 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): return Default_Validations(test_result, durationInMs, expectedLatencyMs) def set(test_result, durationInMs=0,expectedLatencyMs=0): - dab_response_validator.validate_set_voice_system_response_schema(test_result.response) + try: + dab_response_validator.validate_set_voice_system_response_schema(test_result.response) + except Exception as error: + print("Schema error:", error) + return False response = jsons.loads(test_result.response) if response['status'] != 200: return False From e4a519b1d467f64be819ceeb626502db9101b772 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 15 Jan 2024 13:46:06 -0800 Subject: [PATCH 02/11] Change test logic based on list response --- conformance.py | 6 +-- dab/applications.py | 2 + dab/operations.py | 2 + dab/system.py | 14 ++++++- dab/voice.py | 2 + end_to_end_cobalt.py | 4 +- requirements.txt | 3 +- util/enforcement_manager.py | 83 +++++++++++++++++++++++++++++++++++++ voice_audio.py | 20 +++++---- voice_text.py | 20 +++++---- 10 files changed, 131 insertions(+), 25 deletions(-) create mode 100644 util/enforcement_manager.py diff --git a/conformance.py b/conformance.py index 4da4bd8..486546f 100644 --- a/conformance.py +++ b/conformance.py @@ -10,6 +10,7 @@ import dab.system import dab.output import dab.version +from util.enforcement_manager import EnforcementManager # Implement the test cases for conformance test. CONFORMANCE_TEST_CASE = [ @@ -120,9 +121,8 @@ ("health-check/get",'{}', dab.health_check.get, 2000, "Conformance"), ("voice/list",'{}', dab.voice.list, 200, "Conformance"), ("voice/set",'{}', dab.voice.set, 5000, "Conformance"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/ladygaga.wav"}',dab.voice.send_audio, 10000, "Conformance"), - ("voice/send-text",'{"requestText" : "Play lady Gaga music on YouTube"}', dab.voice.send_text, 10000, "Conformance"), - ("voice/send-text",'{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "Alexa"}', dab.voice.send_text, 10000, "Conformance With VA"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/ladygaga.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}',dab.voice.send_audio, 10000, "Conformance"), + ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, 10000, "Conformance With VA"), ("version",' {}', dab.version.default, 200, "Conformance"), ("system/restart",' {}', dab.system.restart, 30000, "Conformance"), ] \ No newline at end of file diff --git a/dab/applications.py b/dab/applications.py index 11936b1..2aa672b 100644 --- a/dab/applications.py +++ b/dab/applications.py @@ -1,6 +1,7 @@ from time import sleep from schema import dab_response_validator from dab_tester import YesNoQuestion, Default_Validations +from util.enforcement_manager import EnforcementManager import jsons def launch(test_result, durationInMs=0,expectedLatencyMs=0): @@ -49,6 +50,7 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): response = jsons.loads(test_result.response) if response['status'] != 200: return False + EnforcementManager().add_supported_application(response.applications) sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) diff --git a/dab/operations.py b/dab/operations.py index 191549b..21e47d8 100644 --- a/dab/operations.py +++ b/dab/operations.py @@ -2,6 +2,7 @@ from time import sleep from dab_tester import YesNoQuestion, Default_Validations import jsons +from util.enforcement_manager import EnforcementManager def list(test_result, durationInMs=0,expectedLatencyMs=0): try: @@ -12,5 +13,6 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): response = jsons.loads(test_result.response) if response['status'] != 200: return False + EnforcementManager().add_supported_operation(response.operations) sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) diff --git a/dab/system.py b/dab/system.py index 8f45845..dee1fec 100644 --- a/dab/system.py +++ b/dab/system.py @@ -2,6 +2,7 @@ from dab_tester import YesNoQuestion, Default_Validations import jsons from schema import dab_response_validator +from util.enforcement_manager import EnforcementManager def restart(test_result, durationInMs=0,expectedLatencyMs=0): try: @@ -23,7 +24,11 @@ def get(test_result, durationInMs=0,expectedLatencyMs=0): print("Schema error:", error) return False response = jsons.loads(test_result.response) - if response['status'] != 200: + request = jsons.loads(test_result.request) + if not EnforcementManager().is_setting_supported(request.system_setting_key): + if response['status'] != 501: + return False + elif response['status'] != 200: return False sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) @@ -35,7 +40,11 @@ def set(test_result, durationInMs=0,expectedLatencyMs=0): print("Schema error:", error) return False response = jsons.loads(test_result.response) - if response['status'] != 200: + request = jsons.loads(test_result.request) + if not EnforcementManager().is_setting_supported(request.system_setting_key): + if response['status'] != 501: + return False + elif response['status'] != 200: return False sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) @@ -47,6 +56,7 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): print("Schema error:", error) return False response = jsons.loads(test_result.response) + EnforcementManager().set_supported_settings(response) if response['status'] != 200: return False sleep(0.1) diff --git a/dab/voice.py b/dab/voice.py index 99d76a2..e3a8606 100644 --- a/dab/voice.py +++ b/dab/voice.py @@ -2,6 +2,7 @@ from time import sleep from dab_tester import YesNoQuestion, Default_Validations import jsons +from util.enforcement_manager import EnforcementManager def send_audio(test_result, durationInMs=0,expectedLatencyMs=0): try: @@ -42,6 +43,7 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): response = jsons.loads(test_result.response) if response['status'] != 200: return False + EnforcementManager().add_supported_voice_assistant(response.voiceSystems) sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) diff --git a/end_to_end_cobalt.py b/end_to_end_cobalt.py index 7792a91..7da95ab 100644 --- a/end_to_end_cobalt.py +++ b/end_to_end_cobalt.py @@ -3,10 +3,12 @@ import dab.input import dab.voice import config +from util.enforcement_manager import EnforcementManager # Voice action steps END_TO_END_TEST_CASE = [ - ("voice/send-text",'{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "Alexa"}', dab.voice.send_text, "Are you on search page with Lady Gaga?", "End to end launch"), + ("voice/list",'{}', dab.voice.list, 200, "Voice List"), + ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Are you on search page with Lady Gaga?", "End to end launch"), ("input/key-press",'{"keyCode": "KEY_ENTER"}', dab.input.key_press, "Is video playing?", "End to end key press Enter"), ("input/long-key-press",'{"keyCode": "KEY_VOLUME_UP", "durationMs": 3000}', dab.input.long_key_press, "Is volume going up?", "End to end volume up"), ("input/long-key-press",'{"keyCode": "KEY_VOLUME_DOWN", "durationMs": 2000}', dab.input.long_key_press, "Is volume going down?", "End to End volume down"), diff --git a/requirements.txt b/requirements.txt index ae2ec3b..ddc84d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ paho-mqtt readchar FileCache jsons -jsonschema \ No newline at end of file +jsonschema +singleton_decorator \ No newline at end of file diff --git a/util/enforcement_manager.py b/util/enforcement_manager.py new file mode 100644 index 0000000..af51061 --- /dev/null +++ b/util/enforcement_manager.py @@ -0,0 +1,83 @@ +from singleton_decorator import singleton +from typing import List + +class Resolution: + width: int + height: int + frequency: int + +class Settings: + status: int + language: List[str] + outputResolution: List[Resolution] + memc: bool + cec: bool + lowLatencyMode: bool + matchContentFrameRate: List[str] + hdrOutputMode: List[str] + pictureMode: List[str] + audioOutputMode: List[str] + audioOutputSource: List[str] + videoInputSource: List[str] + audioVolume: bool + mute: bool + textToSpeech: bool + +@singleton +class EnforcementManager: + def __init__(self): + self.supported_operations = set() + self.supported_keys = set() + self.supported_voice_assistants = set() + self.supported_settings = None + self.supported_applications = set() + + def add_supported_operation(self, operation): + self.supported_operations.add(operation) + + def is_operation_supported(self, operation): + return not self.supported_operations or operation in self.supported_operations + + def add_supported_key(self, key): + self.supported_keys.add(key) + + def is_key_supported(self, key): + return not self.supported_keys or key in self.supported_keys + + def add_supported_voice_assistant(self, voice_assistant): + self.supported_voice_assistants.add(voice_assistant) + + def is_voice_assistant_supported(self, voice_assistant): + return not self.supported_voice_assistants or voice_assistant in self.supported_voice_assistants + + def get_voice_assistant(self): + return "Alexa" if len(self.supported_voice_assistants) == 0 else self.supported_voice_assistants[0] + + def set_supported_settings(self, settings): + self.supported_settings = settings + + def is_setting_supported(self, setting): + """ + Checks if a setting is supported by the target. + + Args: + setting: The name of the setting. + + Returns: + True if the setting is supported, False otherwise. + """ + + if not self.supported_settings: + return True + + if isinstance(getattr(self.supported_settings, setting), List) and len(getattr(self.supported_settings, setting)) == 0: + return True + + if isinstance(getattr(self.supported_settings, setting), bool) and getattr(self.supported_settings, setting): + return True + + def add_supported_application(self, application): + self.supported_applications.add(application) + + def is_application_supported(self, application): + return not self.supported_applications or application in self.supported_applications \ No newline at end of file diff --git a/voice_audio.py b/voice_audio.py index c733166..60c92ea 100644 --- a/voice_audio.py +++ b/voice_audio.py @@ -1,16 +1,18 @@ import dab.applications import dab.system import dab.voice +from util.enforcement_manager import EnforcementManager # Voice action steps SEND_VOICE_AUDIO_TEST_CASES = [ - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/ladygaga.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Are you on search page with Lady Gaga?", "Voice launch Lady gaga"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/pressenter.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Is video playing?", "Voice play video"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/playvideo.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "If video was not playing, is it playing now?", "Voice resume video"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/setvolume0.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Did volume of the video changed?", "voice mute"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/setvolume5.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Did volume of the video changed?", "voice volume up"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/pausevideo.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Did video paused?", "voice pause"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/fastforwardvideo.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Did video playback fast forward?", "voice fastforward"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/rewindvideo.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Did video playback rewind?", "voice rewind"), - ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/exittomainmenu.wav", "voiceSystem": "Alexa"}', dab.voice.send_audio, "Are you on main menu?", "voice exit"), + ("voice/list",'{}', dab.voice.list, 200, "Voice List"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/ladygaga.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Are you on search page with Lady Gaga?", "Voice launch Lady gaga"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/pressenter.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Is video playing?", "Voice play video"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/playvideo.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "If video was not playing, is it playing now?", "Voice resume video"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/setvolume0.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Did volume of the video changed?", "voice mute"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/setvolume5.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Did volume of the video changed?", "voice volume up"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/pausevideo.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Did video paused?", "voice pause"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/fastforwardvideo.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Did video playback fast forward?", "voice fastforward"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/rewindvideo.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Did video playback rewind?", "voice rewind"), + ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/exittomainmenu.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_audio, "Are you on main menu?", "voice exit"), ] \ No newline at end of file diff --git a/voice_text.py b/voice_text.py index 6da5c7a..8d3e892 100644 --- a/voice_text.py +++ b/voice_text.py @@ -1,16 +1,18 @@ import dab.applications import dab.system import dab.voice +from util.enforcement_manager import EnforcementManager # Voice action steps SEND_VOICE_TEXT_TEST_CASES = [ - ("voice/send-text",'{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "Alexa"}', dab.voice.send_text, "Are you on search page with Lady Gaga?", "Voice launch Lady gaga"), - ("voice/send-text",'{"requestText" : "Press enter", "voiceSystem": "Alexa"}', dab.voice.send_text, "Is video playing?", "Voice play video", "Voice play video"), - ("voice/send-text",'{"requestText" : "Play video", "voiceSystem": "Alexa"}', dab.voice.send_text, "If video was not playing, is it playing now?", "Voice resume video"), - ("voice/send-text",'{"requestText" : "Set volume 0", "voiceSystem": "Alexa"}', dab.voice.send_text, "Did volume of the video changed?", "voice mute"), - ("voice/send-text",'{"requestText" : "Set volume 5", "voiceSystem": "Alexa"}', dab.voice.send_text, "Did volume of the video changed?", "voice volume up"), - ("voice/send-text",'{"requestText" : "Pause Video", "voiceSystem": "Alexa"}', dab.voice.send_text, "Did video paused?", "voice pause"), - ("voice/send-text",'{"requestText" : "Fast forward video", "voiceSystem": "Alexa"}', dab.voice.send_text, "Did video playback fast forward?", "voice fastforward"), - ("voice/send-text",'{"requestText" : "Rewind video", "voiceSystem": "Alexa"}', dab.voice.send_text, "Did video playback rewind?", "voice rewind"), - ("voice/send-text",'{"requestText" : "Exit to main menu", "voiceSystem": "Alexa"}', dab.voice.send_text, "Are you on main menu?", "voice exit"), + ("voice/list",'{}', dab.voice.list, 200, "Voice List"), + ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Are you on search page with Lady Gaga?", "Voice launch Lady gaga"), + ("voice/send-text",f'{{"requestText" : "Press enter", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Is video playing?", "Voice play video", "Voice play video"), + ("voice/send-text",f'{{"requestText" : "Play video", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "If video was not playing, is it playing now?", "Voice resume video"), + ("voice/send-text",f'{{"requestText" : "Set volume 0", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Did volume of the video changed?", "voice mute"), + ("voice/send-text",f'{{"requestText" : "Set volume 5", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Did volume of the video changed?", "voice volume up"), + ("voice/send-text",f'{{"requestText" : "Pause Video", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Did video paused?", "voice pause"), + ("voice/send-text",f'{{"requestText" : "Fast forward video", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Did video playback fast forward?", "voice fastforward"), + ("voice/send-text",f'{{"requestText" : "Rewind video", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Did video playback rewind?", "voice rewind"), + ("voice/send-text",f'{{"requestText" : "Exit to main menu", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Are you on main menu?", "voice exit"), ] \ No newline at end of file From 85e51b8d2423fa9dd445aae02c6b1e76d39b93e0 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 7 Feb 2024 16:16:48 -0800 Subject: [PATCH 03/11] Update dab/applications.py Co-authored-by: Jakub Adam --- dab/applications.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dab/applications.py b/dab/applications.py index 2aa672b..1af80c3 100644 --- a/dab/applications.py +++ b/dab/applications.py @@ -50,7 +50,8 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): response = jsons.loads(test_result.response) if response['status'] != 200: return False - EnforcementManager().add_supported_application(response.applications) + for application in response['applications']: + EnforcementManager().add_supported_application(application['appId']) sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) From 9e3ba8dff723469253fb9653a6ea802ca032fcc7 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 7 Feb 2024 16:17:14 -0800 Subject: [PATCH 04/11] Update dab/system.py Co-authored-by: Jakub Adam --- dab/system.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dab/system.py b/dab/system.py index dee1fec..8a8d3ce 100644 --- a/dab/system.py +++ b/dab/system.py @@ -24,11 +24,7 @@ def get(test_result, durationInMs=0,expectedLatencyMs=0): print("Schema error:", error) return False response = jsons.loads(test_result.response) - request = jsons.loads(test_result.request) - if not EnforcementManager().is_setting_supported(request.system_setting_key): - if response['status'] != 501: - return False - elif response['status'] != 200: + if response['status'] != 200: return False sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) From edfeede07bc578eac0fbcfcdb9bca0e151607817 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 7 Feb 2024 16:17:30 -0800 Subject: [PATCH 05/11] Update util/enforcement_manager.py Co-authored-by: Jakub Adam --- util/enforcement_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/enforcement_manager.py b/util/enforcement_manager.py index af51061..2a1810d 100644 --- a/util/enforcement_manager.py +++ b/util/enforcement_manager.py @@ -51,7 +51,7 @@ def is_voice_assistant_supported(self, voice_assistant): return not self.supported_voice_assistants or voice_assistant in self.supported_voice_assistants def get_voice_assistant(self): - return "Alexa" if len(self.supported_voice_assistants) == 0 else self.supported_voice_assistants[0] + return "AmazonAlexa" if len(self.supported_voice_assistants) == 0 else self.supported_voice_assistants[0] def set_supported_settings(self, settings): self.supported_settings = settings From 3dcd2bea95e57edcf15351e8b9c2985a56bd1700 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 7 Feb 2024 16:19:22 -0800 Subject: [PATCH 06/11] Update dab/operations.py Co-authored-by: Jakub Adam --- dab/operations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dab/operations.py b/dab/operations.py index 21e47d8..c19e854 100644 --- a/dab/operations.py +++ b/dab/operations.py @@ -13,6 +13,7 @@ def list(test_result, durationInMs=0,expectedLatencyMs=0): response = jsons.loads(test_result.response) if response['status'] != 200: return False - EnforcementManager().add_supported_operation(response.operations) + for operation in response['operations']: + EnforcementManager().add_supported_operation(operation) sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) From 995ef79ee7dfb0f811f1e03762bb54c1a8634134 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 12 Feb 2024 14:34:14 -0800 Subject: [PATCH 07/11] Update util/enforcement_manager.py Co-authored-by: Jakub Adam --- util/enforcement_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/enforcement_manager.py b/util/enforcement_manager.py index 2a1810d..fbdd039 100644 --- a/util/enforcement_manager.py +++ b/util/enforcement_manager.py @@ -70,10 +70,10 @@ def is_setting_supported(self, setting): if not self.supported_settings: return True - if isinstance(getattr(self.supported_settings, setting), List) and len(getattr(self.supported_settings, setting)) == 0: + if isinstance(self.supported_settings.get(setting), List) and len(self.supported_settings.get(setting)) == 0: return True - if isinstance(getattr(self.supported_settings, setting), bool) and getattr(self.supported_settings, setting): + if isinstance(self.supported_settings.get(setting), bool) and self.supported_settings.get(setting): return True def add_supported_application(self, application): From 26726048fc08fda9a3f41eee83f40f6987b2054e Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 12 Feb 2024 14:34:31 -0800 Subject: [PATCH 08/11] Update voice_text.py Co-authored-by: Jakub Adam --- voice_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/voice_text.py b/voice_text.py index 8d3e892..6e6646a 100644 --- a/voice_text.py +++ b/voice_text.py @@ -7,7 +7,7 @@ SEND_VOICE_TEXT_TEST_CASES = [ ("voice/list",'{}', dab.voice.list, 200, "Voice List"), ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Are you on search page with Lady Gaga?", "Voice launch Lady gaga"), - ("voice/send-text",f'{{"requestText" : "Press enter", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Is video playing?", "Voice play video", "Voice play video"), + ("voice/send-text",f'{{"requestText" : "Press enter", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Is video playing?", "Voice play video"), ("voice/send-text",f'{{"requestText" : "Play video", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "If video was not playing, is it playing now?", "Voice resume video"), ("voice/send-text",f'{{"requestText" : "Set volume 0", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Did volume of the video changed?", "voice mute"), ("voice/send-text",f'{{"requestText" : "Set volume 5", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Did volume of the video changed?", "voice volume up"), From 02c77d2fad3f023615437712671907d0d65b2311 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 12 Feb 2024 23:44:08 +0000 Subject: [PATCH 09/11] Change file according to suggestion. --- dab/system.py | 9 +++--- schema.py | 81 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/dab/system.py b/dab/system.py index 8a8d3ce..de19535 100644 --- a/dab/system.py +++ b/dab/system.py @@ -37,10 +37,11 @@ def set(test_result, durationInMs=0,expectedLatencyMs=0): return False response = jsons.loads(test_result.response) request = jsons.loads(test_result.request) - if not EnforcementManager().is_setting_supported(request.system_setting_key): - if response['status'] != 501: - return False - elif response['status'] != 200: + for setting in request: + if not EnforcementManager().is_setting_supported(setting): + if response['status'] != 501: + return False + if response['status'] != 200: return False sleep(0.1) return Default_Validations(test_result, durationInMs, expectedLatencyMs) diff --git a/schema.py b/schema.py index da5f1d1..36366e5 100644 --- a/schema.py +++ b/schema.py @@ -273,44 +273,75 @@ "audioOutputSource", "videoInputSource", "audioVolume", "mute", "textToSpeech"] } +system_settings_schema = { + "language": {"type": "string"}, + "outputResolution": output_resolution_schema, + "memc": {"type": "boolean"}, + "cec": {"type": "boolean"}, + "lowLatencyMode": {"type": "boolean"}, + "matchContentFrameRate": {"type": "string"}, + "hdrOutputMode": {"type": "string"}, + "pictureMode": {"type": "string"}, + "audioOutputMode": {"type": "string"}, + "audioOutputSource": {"type": "string"}, + "videoInputSource": {"type": "string"}, + "audioVolume": {"type": "integer"}, + "mute": {"type": "boolean"}, + "textToSpeech": {"type": "boolean"}, +} + # Operation: system/settings/get # GetSystemSettingsResponse get_system_settings_response_schema = { "type": "object", "properties": { - "status": {"type": "integer"}, - "error": {"type": ["string", "null"]}, - "language": {"type": "string"}, - "outputResolution": output_resolution_schema, - "memc": {"type": "boolean"}, - "cec": {"type": "boolean"}, - "lowLatencyMode": {"type": "boolean"}, - "matchContentFrameRate": {"type": "string"}, - "hdrOutputMode": {"type": "string"}, - "pictureMode": {"type": "string"}, - "audioOutputMode": {"type": "string"}, - "audioOutputSource": {"type": "string"}, - "videoInputSource": {"type": "string"}, - "audioVolume": {"type": "integer"}, - "mute": {"type": "boolean"}, - "textToSpeech": {"type": "boolean"}, + "status": {"type": "integer"} }, - "required": ["status", "language", "outputResolution", "memc", "cec", "lowLatencyMode", - "matchContentFrameRate", "hdrOutputMode", "pictureMode", "audioOutputMode", - "audioOutputSource", "videoInputSource", "audioVolume", "mute", "textToSpeech"] + "required": ["status"], + + "if": { + "properties": { + "status": { "const": 200 } + } + }, + "then": { + "properties": system_settings_schema, + "required": system_settings_schema.keys() + }, + "else": { + "properties": { + "error": {"type": ["string", "null"]} + }, + "required": ["error"] + }, + + "unevaluatedProperties": False } # SetSystemSettingsResponse set_system_settings_response_schema = { "type": "object", "properties": { - "status": {"type": "integer"}, - "error": {"type": ["string", "null"]}, - "system_setting_key": {"type": "string"}, - "value": {}, + "status": {"type": "integer"} }, - "required": ["status", "system_setting_key", "value"] -} + "required": ["status"], + + "if": { + "properties": { + "status": { "const": 200 } + } + }, + "then": { + "properties": system_settings_schema + }, + "else": { + "properties": { + "error": {"type": ["string", "null"]} + }, + "required": ["error"] + }, + + "unevaluatedProperties": False # Operation: input/key/list # KeyList From 3e6cf73fc77bbef89e276e795d0bef8a9a94b75f Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 12 Feb 2024 18:52:20 -0800 Subject: [PATCH 10/11] Update schema.py Co-authored-by: Jakub Adam --- schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/schema.py b/schema.py index 36366e5..d3a583f 100644 --- a/schema.py +++ b/schema.py @@ -342,6 +342,7 @@ }, "unevaluatedProperties": False +} # Operation: input/key/list # KeyList From 4bcab0c071254785e1cbeb6d9607d857de983128 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 4 Mar 2024 21:38:19 -0800 Subject: [PATCH 11/11] Update schema.py Co-authored-by: Jakub Adam --- schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.py b/schema.py index d3a583f..28f3b2f 100644 --- a/schema.py +++ b/schema.py @@ -306,7 +306,7 @@ }, "then": { "properties": system_settings_schema, - "required": system_settings_schema.keys() + "required": list(system_settings_schema.keys()) }, "else": { "properties": {