Merge branch 'main' into TestLogic

.gitignore

@@ -1,2 +1,4 @@
 __pycache__/
-.DA_Store
+.DS_Store
+screenshot.html
+test_result/output_image.json

config.py

@@ -2,5 +2,8 @@
 apps = dict(
     youtube = 'YouTube',
     netflix = 'Netflix',
-    amazon = 'primeLive'
-)
+    amazon = 'PrimeVideo'
+)
+
+# Please set the voice asssitant you want to set here.
+va = 'AmazonAlexa'

conformance.py

@@ -17,7 +17,7 @@
     ("operations/list",'{}', dab.operations.list, 200, "Conformance"),
     ("applications/list",'{}', dab.applications.list, 200, "Conformance"),
     ("applications/launch",f'{{"appId": "{config.apps["youtube"]}"}}', dab.applications.launch, 10000, "Conformance"),
-    ("applications/launch",f'{{"appId": "{config.apps["youtube"]}", "parameters": ["#/watch?list=OLAK5uy_mKAu6VNK3gMSq_L8fU_C6myQnuuuIzvWY"]}}', dab.applications.launch, 10000, "with parameters"),
+    ("applications/launch",f'{{"appId": "{config.apps["youtube"]}", "parameters": ["v%3DSs75O8yllyc","enableEventConsole%3Dtrue","env_showConsole%3Dtrue"]}}', dab.applications.launch, 10000, "with parameters"),
     ("applications/launch-with-content",f'{{"appId": "{config.apps["youtube"]}", "contentId": "jfKfPfyJRdk"}}', dab.applications.launch_with_content, 10000, "Conformance"),
     ("applications/get-state",f'{{"appId": "{config.apps["youtube"]}"}}', dab.applications.get_state, 200, "Conformance"),
     ("applications/exit",f'{{"appId": "{config.apps["youtube"]}"}}', dab.applications.exit, 5000, "Conformance"),
@@ -34,7 +34,7 @@
     ("system/settings/set",'{"pictureMode": "Standard"}', dab.system.set, 3000, "pictureMode"),
     ("system/settings/set",'{"audioOutputMode": "Auto"}', dab.system.set, 3000, "audioOutputMode"),
     ("system/settings/set",'{"audioOutputSource": "HDMI"}', dab.system.set, 3000, "audioOutputSource"),
-    ("system/settings/set",'{"videoInputSource": "Other"}', dab.system.set, 3000, "videoInputSource"),
+    ("system/settings/set",'{"videoInputSource": "HDMI1"}', dab.system.set, 3000, "videoInputSource"),
     ("system/settings/set",'{"audioVolume": 20}', dab.system.set, 3000, "audioVolume"),
     ("system/settings/set",'{"mute": false}', dab.system.set, 3000, "mute"),
     ("system/settings/set",'{"textToSpeech": true}', dab.system.set, 3000, "textToSpeech"),
@@ -114,15 +114,15 @@
     ("input/long-key-press",'{"keyCode": "KEY_8", "durationMs": 3000}', dab.input.long_key_press, 5000, "KEY_8"),
     ("input/long-key-press",'{"keyCode": "KEY_9", "durationMs": 3000}', dab.input.long_key_press, 5000, "KEY_9"),
     ("output/image",'{}', dab.output.image, 2000, "Conformance"),
-    ("device-telemetry/start",'{"durationMs": 1000}', dab.device_telemetry.start, 200, "Conformance"),
+    ("device-telemetry/start",'{"duration": 1000}', dab.device_telemetry.start, 200, "Conformance"),
     ("device-telemetry/stop",'{}', dab.device_telemetry.stop, 200, "Conformance"),
-    ("app-telemetry/start",f'{{"appId": "{config.apps["youtube"]}", "durationMs": 1000}}', dab.app_telemetry.start, 200, "Conformance"),
+    ("app-telemetry/start",f'{{"appId": "{config.apps["youtube"]}", "duration": 1000}}', dab.app_telemetry.start, 200, "Conformance"),
     ("app-telemetry/stop",f'{{"appId": "{config.apps["youtube"]}"}}', dab.app_telemetry.stop, 200, "Conformance"),
     ("health-check/get",'{}', dab.health_check.get, 2000, "Conformance"),
     ("voice/list",'{}', dab.voice.list, 200, "Conformance"),
-    ("voice/set",'{}', dab.voice.set, 5000, "Conformance"),
+    ("voice/set",f'{{"voiceSystem":{{"name":"{EnforcementManager().get_voice_assistant()}","enabled":true}}}}', dab.voice.set, 5000, "Conformance"),
     ("voice/send-audio",f'{{"fileLocation": "https://storage.googleapis.com/ytlr-cert.appspot.com/voice/ladygaga.wav", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}',dab.voice.send_audio, 10000, "Conformance"),
     ("voice/send-text",f'{{"requestText" : "Play lady Gaga music on YouTube", "voiceSystem": "{EnforcementManager().get_voice_assistant()}"}}', dab.voice.send_text, 10000, "Conformance With VA"),
     ("version",' {}', dab.version.default, 200, "Conformance"),
     ("system/restart",' {}', dab.system.restart, 30000, "Conformance"),
-]
+]

dab/applications.py

@@ -14,7 +14,7 @@ def launch(test_result, durationInMs=0,expectedLatencyMs=0):
     if response['status'] != 200:
         return False
     sleep(5)
-    return YesNoQuestion(test_result, "Cobalt started?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
+    return YesNoQuestion(test_result, "App started?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def launch_with_content(test_result, durationInMs=0,expectedLatencyMs=0):
     dab_response_validator.validate_dab_response_schema(test_result.response)
@@ -27,7 +27,7 @@ def launch_with_content(test_result, durationInMs=0,expectedLatencyMs=0):
     if response['status'] != 200:
         return False
     sleep(5)
-    return YesNoQuestion(test_result, "Cobalt started with playback?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
+    return YesNoQuestion(test_result, "App started with playback?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def exit(test_result, durationInMs=0,expectedLatencyMs=0):
     try:
@@ -39,7 +39,7 @@ def exit(test_result, durationInMs=0,expectedLatencyMs=0):
     if response['status'] != 200:
         return False
     sleep(5)
-    return YesNoQuestion(test_result, "Cobalt exited?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
+    return YesNoQuestion(test_result, "App exited?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
 
 def list(test_result, durationInMs=0,expectedLatencyMs=0):
     try:

dab/output.py

@@ -12,5 +12,9 @@ def image(test_result, durationInMs=0,expectedLatencyMs=0):
     response  = jsons.loads(test_result.response)
     if response['status'] != 200:
         return False
-    sleep(5)
-    return YesNoQuestion(test_result, f"Can you verify the image has been uploaded?") and Default_Validations(test_result, durationInMs, expectedLatencyMs)
+
+    SCREENSHOT_FILENAME = "screenshot.html"
+
+    with open(SCREENSHOT_FILENAME, "w") as outfile:
+        outfile.write('<div><img src="' + response['outputImage'] + '"</div>')
+    return YesNoQuestion(test_result, f"Verify if the {SCREENSHOT_FILENAME} file was created and contains the screenshot") and Default_Validations(test_result, durationInMs, expectedLatencyMs)

dab_tester.py

@@ -30,11 +30,20 @@ def Execute_Test_Case(self, device_id, test_case):
             end = datetime.datetime.now()
             duration = end - start
             durationInMs = int(duration.total_seconds() * 1000)
-            if validate_output_function(test_result, durationInMs, expected_response) == True:
+            exception = None
+            try:
+                validate_result = validate_output_function(test_result, durationInMs, expected_response)
+            except Exception as e:
+                validate_result = False
+                exception = e
+
+            if validate_result == True:
                 log(test_result, "\033[1;32m[ PASS ]\033[0m")
                 test_result.test_result = "PASS"
             else:
                 log(test_result, "\033[1;31m[ FAILED ]\033[0m")
+                if exception:
+                    log(test_result, f"{type(exception).__name__} raised during result validation:\n\033[0;31m{exception}\033[0m\n")
                 test_result.test_result = "FAILED"
         else:
             log(test_result, '\033[1;31m[ ')

main.py

@@ -18,15 +18,21 @@
 import end_to_end_cobalt
 import voice_audio
 import voice_text
+import output_image
 
 ALL_SUITES = {
     "conformance": conformance.CONFORMANCE_TEST_CASE,
     "end_to_end_cobalt": end_to_end_cobalt.END_TO_END_TEST_CASE,
     "voice_audio": voice_audio.SEND_VOICE_AUDIO_TEST_CASES,
-    "voice_text": voice_text.SEND_VOICE_TEXT_TEST_CASES
+    "voice_text": voice_text.SEND_VOICE_TEXT_TEST_CASES,
+    "output_image": output_image.OUTPUT_IMAGE_TEST_CASES
 }
 
 if __name__ == "__main__":
+    test_suites_str = ""
+    for field_name in ALL_SUITES.keys():
+        test_suites_str += field_name + ", "
+    test_suites_str = test_suites_str[:-2]
     parser = argparse.ArgumentParser()
     parser.add_argument("-v","--verbose", 
                         help="increase output verbosity",
@@ -55,7 +61,7 @@
                         type=str)
 
     parser.add_argument("-s","--suite",
-                        help="set what test suite to run. Avaible test suite includes: conformance, voice_audio, voice_text, end_to_end_cobalt",
+                        help="set what test suite to run. Available test suite includes:" + test_suites_str,
                         type=str)
 
     parser.set_defaults(output="")

output_image.py

@@ -0,0 +1,6 @@
+import dab.output
+
+# Voice action steps
+OUTPUT_IMAGE_TEST_CASES = [
+    ("output/image",'{}', dab.output.image, 10000, "Output Image"),
+]

requirements.txt

@@ -1,5 +1,5 @@
 ###### Requirements without Version Specifiers ######
-paho-mqtt
+paho-mqtt == 1.6.1
 readchar
 FileCache
 jsons