Reboot Android Emulator on test errors

.github/workflows/integration_tests.yml

@@ -750,7 +750,7 @@ jobs:
         shell: bash
         run: |
           if [ ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then
-            echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json
+            mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json
           fi
       - name: Upload Desktop test results artifact
         if: ${{ !cancelled() }}
@@ -836,7 +836,7 @@ jobs:
         shell: bash
         run: |
           if [ ! -f "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json" ]; then
-            echo "__SUMMARY_MISSING__" > "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json"
+            mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json"
           fi
       - name: Upload Android test results artifact
         if: ${{ !cancelled() }}
@@ -921,7 +921,7 @@ jobs:
         shell: bash
         run: |
           if [ ! -f "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log.json" ]; then
-            echo "__SUMMARY_MISSING__" > "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log.json"
+            mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log.json"
           fi
       - name: Upload iOS test results artifact
         if: ${{ !cancelled() }}
@@ -990,7 +990,7 @@ jobs:
         shell: bash
         run: |
           if [ ! -f "testapps/test-results-tvos-macos-latest-${{ matrix.tvos_device }}.log.json" ]; then
-            echo "__SUMMARY_MISSING__" > "testapps/test-results-tvos-macos-latest-${{ matrix.tvos_device }}.log.json"
+            mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-tvos-macos-latest-${{ matrix.tvos_device }}.log.json"
           fi
       - name: Upload tvOS test results artifact
         if: ${{ !cancelled() }}

scripts/gha/it_workflow.py

@@ -62,7 +62,7 @@
 _COMMENT_TITLE_FAIL = "### ❌  Integration test FAILED\n"
 _COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n"
 
-_COMMENT_FLAKY_TRACKER = "\nAdd flaky tests to **[go/fpl-cpp-flake-tracker](http://go/fpl-cpp-flake-tracker)**\n"
+_COMMENT_FLAKY_TRACKER = "\n\nAdd flaky tests to **[go/fpl-cpp-flake-tracker](http://go/fpl-cpp-flake-tracker)**\n"
 
 _COMMENT_IDENTIFIER = "integration-test-status-comment"
 _COMMENT_SUFFIX = f'\n\n\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>'
@@ -130,6 +130,7 @@ def test_progress(token, issue_number, actor, commit, run_id):
     comment = (_COMMENT_TITLE_PROGESS_FAIL +
                _get_description(actor, commit, run_id) +
                log_summary +
+               _COMMENT_FLAKY_TRACKER +
                _COMMENT_SUFFIX)
     _update_comment(token, issue_number, comment)
 
@@ -150,6 +151,7 @@ def test_end(token, issue_number, actor, commit, run_id, new_token):
     comment = (_COMMENT_TITLE_FAIL +
                _get_description(actor, commit, run_id) +
                log_summary +
+               _COMMENT_FLAKY_TRACKER +
                _COMMENT_SUFFIX)
     _update_comment(token, issue_number, comment)
 

scripts/gha/test_lab.py

@@ -278,6 +278,7 @@ def run(self):
       self.raw_result_link = raw_result_link.group(1)
 
     self.logs = self._get_testapp_log_text_from_gcs()
+    logging.info("Test result: %s", self.logs)
 
   @property
   def _gcloud_command(self):

scripts/gha/test_simulator.py

@@ -461,7 +461,9 @@ def _get_apple_test_log(bundle_id, app_path, device_id):
     return None
 
   log_path = os.path.join(result.stdout.strip(), "Documents", "GameLoopResults", _RESULT_FILE) 
-  return _read_file(log_path) 
+  log = _read_file(log_path) 
+  logging.info("Apple test result: %s", log)
+  return log
 
 
 def _read_file(path):
@@ -501,12 +503,9 @@ def _setup_android(platform_version, build_tool_version, sdk_id):
   logging.info("Install packages: %s", " ".join(args))
   subprocess.run(args=args, check=True)
 
-  args = ["sdkmanager", "--licenses"]
-  logging.info("Accept all licenses: %s", " ".join(args))
-  p_yes = subprocess.Popen(["echo", "yes"], stdout=subprocess.PIPE)
-  proc = subprocess.Popen(args, stdin=p_yes.stdout, stdout=subprocess.PIPE)
-  p_yes.stdout.close()
-  proc.communicate()
+  command = "yes | sdkmanager --licenses"
+  logging.info("Accept all licenses: %s", command)
+  subprocess.run(command, shell=True, check=False)
 
   args = ["sdkmanager", sdk_id]
   logging.info("Download an emulator: %s", " ".join(args))
@@ -521,25 +520,24 @@ def _shutdown_emulator():
   command = "adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done"
   logging.info("Kill all running emulator: %s", command)
   subprocess.Popen(command, universal_newlines=True, shell=True, stdout=subprocess.PIPE)
+  time.sleep(5)
+  args = ["adb", "kill-server"]
+  logging.info("Kill adb server: %s", " ".join(args))
+  subprocess.run(args=args, check=False)
+  time.sleep(5)
+
 
 def _create_and_boot_emulator(sdk_id):
-  args = ["avdmanager", "-s", 
-    "create", "avd", 
-    "-n", "test_emulator", 
-    "-k", sdk_id, 
-    "-f"]
-  logging.info("Create an emulator: %s", " ".join(args))
-  p_no = subprocess.Popen(["echo", "no"], stdout=subprocess.PIPE)
-  proc = subprocess.Popen(args, stdin=p_no.stdout, stdout=subprocess.PIPE)
-  p_no.stdout.close()
-  proc.communicate()
+  _shutdown_emulator()
+
+  command = "echo no | avdmanager -s create avd -n test_emulator -k '%s' -f" % sdk_id
+  logging.info("Create an emulator: %s", command)
+  subprocess.run(command, shell=True, check=True)
 
   args = ["adb", "start-server"]
   logging.info("Start adb server: %s", " ".join(args))
   subprocess.run(args=args, check=True)
 
-  _shutdown_emulator()
-
   if not FLAGS.ci: 
     command = "$ANDROID_HOME/emulator/emulator -avd test_emulator &"
   else:
@@ -550,14 +548,30 @@ def _create_and_boot_emulator(sdk_id):
   args = ["adb", "wait-for-device"]
   logging.info("Wait for emulator to boot: %s", " ".join(args))
   subprocess.run(args=args, check=True)
-
   if FLAGS.ci: 
     # wait extra 90 seconds to ensure emulator booted.
     time.sleep(90)
   else:
     time.sleep(45)
 
 
+def _reset_emulator_on_error(instrumented_test_result):
+  logging.info("game-loop test result: %s", instrumented_test_result)
+  if "FAILURES!!!" in instrumented_test_result:
+    logging.info("game-loop test error!!! reboot emualtor...")
+    args = ["adb", "-e", "reboot"]
+    logging.info("Reboot android emulator: %s", " ".join(args))
+    subprocess.run(args=args, check=True)
+    args = ["adb", "wait-for-device"]
+    logging.info("Wait for emulator to boot: %s", " ".join(args))
+    subprocess.run(args=args, check=True)
+    if FLAGS.ci: 
+      # wait extra 90 seconds to ensure emulator booted.
+      time.sleep(90)
+    else:
+      time.sleep(45)
+
+
 def _get_package_name(app_path):
   command = "aapt dump badging %s | awk -v FS=\"'\" '/package: name=/{print $2}'" % app_path
   logging.info("Get package_name: %s", command)
@@ -597,6 +611,11 @@ def _uninstall_android_app(package_name):
 
 def _install_android_gameloop_app(gameloop_project):
   os.chdir(gameloop_project)
+  logging.info("CD to gameloop_project: %s", gameloop_project)
+  _uninstall_android_app("com.google.firebase.gameloop")
+  args = ["./gradlew", "clean"]
+  logging.info("Clean game-loop cache: %s", " ".join(args))
+  subprocess.run(args=args, check=False)
   args = ["./gradlew", "installDebug", "installDebugAndroidTest"]
   logging.info("Installing game-loop app and test: %s", " ".join(args))
   subprocess.run(args=args, check=True)
@@ -609,7 +628,8 @@ def _run_instrumented_test():
   args = ["adb", "shell", "am", "instrument",
     "-w", "%s.test/androidx.test.runner.AndroidJUnitRunner" % _GAMELOOP_PACKAGE] 
   logging.info("Running game-loop test: %s", " ".join(args))
-  subprocess.run(args=args, check=False) 
+  result = subprocess.run(args=args, capture_output=True, text=True, check=False) 
+  _reset_emulator_on_error(result.stdout)
 
 
 def _get_android_test_log(test_package):
@@ -619,6 +639,7 @@ def _get_android_test_log(test_package):
   args = ["adb", "shell", "su", "0", "cat", path]
   logging.info("Get android test result: %s", " ".join(args))
   result = subprocess.run(args=args, capture_output=True, text=True, check=False)
+  logging.info("Android test result: %s", result.stdout)
   return result.stdout