Merge pull request #42 from device-automation-bus/TestLogic

Test logic
device-automation-bus · Mar 12, 2024 · 5adf476 · 5adf476
2 parents 7702f9c + 4bcab0c
commit 5adf476
Show file tree

Hide file tree

Showing 18 changed files with 311 additions and 86 deletions.
diff --git a/conformance.py b/conformance.py
@@ -10,6 +10,7 @@
 import dab.system
 import dab.output
 import dab.version
+from util.enforcement_manager import EnforcementManager
 
 # Implement the test cases for conformance test. 
 CONFORMANCE_TEST_CASE = [
@@ -119,9 +120,9 @@
     ("app-telemetry/stop",f'{{"appId": "{config.apps["youtube"]}"}}', dab.app_telemetry.stop, 200, "Conformance"),
     ("health-check/get",'{}', dab.health_check.get, 2000, "Conformance"),
     ("voice/list",'{}', dab.voice.list, 200, "Conformance"),
-    ("voice/set",f'{{"voiceSystem":{{"name":"{config.va}","enabled":true}}}}', dab.voice.set, 5000, "Conformance"),
-    ("voice/send-audio",'{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/ladygaga.wav"}',dab.voice.send_audio, 10000, "Conformance"),
-    ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem" : "{config.va}"}}', dab.voice.send_text, 10000, "Conformance"),
+    ("voice/set",f'{{"voiceSystem":{{"name":"{EnforcementManager().get_voice_assistant()}","enabled":true}}}}', dab.voice.set, 5000, "Conformance"),
+    ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/ladygaga.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}',dab.voice.send_audio, 10000, "Conformance"),
+    ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, 10000, "Conformance With VA"),
     ("version",' {}', dab.version.default, 200, "Conformance"),
     ("system/restart",' {}', dab.system.restart, 30000, "Conformance"),
 ]
diff --git a/dab/app_telemetry.py b/dab/app_telemetry.py
@@ -5,14 +5,22 @@
 
 
 def start(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
     sleep(0.1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def stop(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     sleep(0.1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
diff --git a/dab/applications.py b/dab/applications.py
@@ -1,10 +1,15 @@
 from time import sleep
 from schema import dab_response_validator
 from dab_tester import YesNoQuestion, Default_Validations
+from util.enforcement_manager import EnforcementManager
 import jsons
 
 def launch(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_dab_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_dab_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
@@ -13,30 +18,49 @@ def launch(test_result, durationInMs=0,expectedLatencyMs=0):
 
 def launch_with_content(test_result, durationInMs=0,expectedLatencyMs=0):
     dab_response_validator.validate_dab_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_dab_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
     sleep(5)
     return YesNoQuestion(test_result, "App started with playback?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def exit(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_exit_application_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_exit_application_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
     sleep(5)
     return YesNoQuestion(test_result, "App exited?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def list(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_list_applications_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_list_applications_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
+    for application in response['applications']:
+        EnforcementManager().add_supported_application(application['appId'])
     sleep(0.1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def get_state(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_get_application_state_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_get_application_state_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/dab/device.py b/dab/device.py
@@ -4,7 +4,11 @@
 import jsons
 
 def info(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_device_information_schema(test_result.response)
+    try:
+        dab_response_validator.validate_device_information_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/dab/device_telemetry.py b/dab/device_telemetry.py
@@ -4,15 +4,23 @@
 import jsons
 
 def start(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_start_device_telemetry_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
     sleep(1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def stop(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_stop_device_telemetry_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/dab/health_check.py b/dab/health_check.py
@@ -4,7 +4,11 @@
 import jsons
 
 def get(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_health_check_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_health_check_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/dab/input.py b/dab/input.py
@@ -7,7 +7,11 @@ class KeyList:
     key_list = []
 
 def key_press(test_result, durationInMs=0, expectedLatencyMs=None):
-    dab_response_validator.validate_dab_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_dab_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     request = jsons.loads(test_result.request)
     response  = jsons.loads(test_result.response)
     # No list available, assuming everything is required.
@@ -28,7 +32,11 @@ def key_press(test_result, durationInMs=0, expectedLatencyMs=None):
         return YesNoQuestion(test_result, expectedLatencyMs)
 
 def long_key_press(test_result, durationInMs=0, expectedLatencyMs=None):
-    dab_response_validator.validate_dab_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_dab_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     request = jsons.loads(test_result.request)
     response  = jsons.loads(test_result.response)
     # No list available, assuming everything is required.
@@ -50,7 +58,11 @@ def long_key_press(test_result, durationInMs=0, expectedLatencyMs=None):
         return YesNoQuestion(test_result, expectedLatencyMs)
 
 def list(test_result, durationInMs=0, expectedLatencyMs=0):
-    dab_response_validator.validate_key_list_schema(test_result.response)
+    try:
+        dab_response_validator.validate_key_list_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/dab/operations.py b/dab/operations.py
@@ -2,11 +2,18 @@
 from time import sleep
 from dab_tester import YesNoQuestion, Default_Validations
 import jsons
+from util.enforcement_manager import EnforcementManager
 
 def list(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_list_supported_operation_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_list_supported_operation_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
+    for operation in response['operations']:
+        EnforcementManager().add_supported_operation(operation)
     sleep(0.1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
diff --git a/dab/output.py b/dab/output.py
@@ -4,7 +4,11 @@
 import jsons
 
 def image(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_output_image_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_output_image_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/dab/system.py b/dab/system.py
@@ -2,9 +2,14 @@
 from dab_tester import YesNoQuestion, Default_Validations
 import jsons
 from schema import dab_response_validator
+from util.enforcement_manager import EnforcementManager
 
 def restart(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_dab_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_dab_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
@@ -13,24 +18,42 @@ def restart(test_result, durationInMs=0,expectedLatencyMs=0):
     return YesNoQuestion(test_result, "Cobalt re-started?")
 
 def get(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_get_system_settings_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_get_system_settings_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
     sleep(0.1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def set(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_set_system_settings_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_set_system_settings_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
+    request = jsons.loads(test_result.request)
+    for setting in request:
+        if not EnforcementManager().is_setting_supported(setting):
+            if response['status'] != 501:
+                return False
     if response['status'] != 200:
         return False
     sleep(0.1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def list(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_list_system_settings_schema(test_result.response)
+    try:
+        dab_response_validator.validate_list_system_settings_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
+    EnforcementManager().set_supported_settings(response)
     if response['status'] != 200:
         return False
     sleep(0.1)

diff --git a/dab/version.py b/dab/version.py
@@ -4,7 +4,11 @@
 import jsons
 
 def default(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_version_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_version_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/dab/voice.py b/dab/voice.py
@@ -2,9 +2,14 @@
 from time import sleep
 from dab_tester import YesNoQuestion, Default_Validations
 import jsons
+from util.enforcement_manager import EnforcementManager
 
 def send_audio(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_dab_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_dab_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
@@ -15,7 +20,11 @@ def send_audio(test_result, durationInMs=0,expectedLatencyMs=0):
         return YesNoQuestion(test_result, expectedLatencyMs)
 
 def send_text(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_dab_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_dab_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
@@ -26,15 +35,24 @@ def send_text(test_result, durationInMs=0,expectedLatencyMs=0):
         return YesNoQuestion(test_result, expectedLatencyMs)
 
 def list(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_list_voice_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_list_voice_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
+    EnforcementManager().add_supported_voice_assistant(response.voiceSystems)
     sleep(0.1)
     return Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def set(test_result, durationInMs=0,expectedLatencyMs=0):
-    dab_response_validator.validate_set_voice_system_response_schema(test_result.response)
+    try:
+        dab_response_validator.validate_set_voice_system_response_schema(test_result.response)
+    except Exception as error:
+        print("Schema error:", error)
+        return False
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False

diff --git a/end_to_end_cobalt.py b/end_to_end_cobalt.py
@@ -3,10 +3,12 @@
 import dab.input
 import dab.voice
 import config
+from util.enforcement_manager import EnforcementManager
 
 # Voice action steps
 END_TO_END_TEST_CASE = [
-    ("voice/send-text",'{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "Alexa"}', dab.voice.send_text, "Are you on search page with Lady Gaga?", "End to end launch"),
+    ("voice/list",'{}', dab.voice.list, 200, "Voice List"),
+    ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, "Are you on search page with Lady Gaga?", "End to end launch"),
     ("input/key-press",'{"keyCode": "KEY_ENTER"}', dab.input.key_press, "Is video playing?", "End to end key press Enter"),
     ("input/long-key-press",'{"keyCode": "KEY_VOLUME_UP", "durationMs": 3000}', dab.input.long_key_press, "Is volume going up?", "End to end volume up"),
     ("input/long-key-press",'{"keyCode": "KEY_VOLUME_DOWN", "durationMs": 2000}', dab.input.long_key_press, "Is volume going down?", "End to End volume down"),

diff --git a/requirements.txt b/requirements.txt
@@ -3,4 +3,5 @@ paho-mqtt == 1.6.1
 readchar
 FileCache
 jsons
-jsonschema
+jsonschema
+singleton_decorator