Merge pull request #117 from launchableinc/bazel-test

Fix Bazel plugin

launchable/test_runners/bazel.py

@@ -21,7 +21,7 @@ def subset(client):
         if label.startswith('//'):
             pkg, target = label.rstrip('\n').split(':')
             # TODO: error checks and more robustness
-            client.test_path(make_test_path(pkg, target))
+            client.test_path(make_test_path(pkg.lstrip('//'), target))
 
     client.formatter = lambda x: x[0]['name'] + ":" + x[1]['name']
     client.run()

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test1/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest1' timestamp='2020-12-16T03:11:43.526Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.02' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest1' time='2.02' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test2/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest2' timestamp='2020-12-16T03:11:43.179Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.017' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest2' time='2.017' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test3/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest3' timestamp='2020-12-16T03:11:42.765Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.022' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest3' time='2.022' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test4/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest4' timestamp='2020-12-16T03:11:43.716Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.014' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest4' time='2.014' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test5/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest5' timestamp='2020-12-16T03:11:43.361Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.017' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest5' time='2.017' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test6/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest6' timestamp='2020-12-16T03:11:43.009Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.018' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest6' time='2.018' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test7/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest7' timestamp='2020-12-16T03:11:42.809Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.018' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest7' time='2.018' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test8/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest8' timestamp='2020-12-16T03:11:42.885Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.019' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest8' time='2.019' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test9/test.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+  <testsuite name='com.ninjinkun.MyLibTest9' timestamp='2020-12-16T03:11:43.867Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.015' package='' id='0'>
+    <properties />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest9' time='2.015' />
+    <system-out />
+    <system-err /></testsuite></testsuites>

tests/data/bazel/record_test_result.json

@@ -0,0 +1,130 @@
+{
+  "events": [
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test6" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest6" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.018,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test1" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest1" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.02,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test8" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest8" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.019,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test9" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest9" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.015,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test7" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest7" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.018,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test2" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest2" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.017,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test5" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest5" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.017,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test4" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest4" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.014,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        { "type": "package", "name": "src/test/java/com/ninjinkun" },
+        { "type": "target", "name": "mylib_test3" },
+        { "type": "class", "name": "com.ninjinkun.MyLibTest3" },
+        { "type": "testcase", "name": "testFibonacci", "_lineno": null }
+      ],
+      "duration": 2.022,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    }
+  ]
+}

tests/data/bazel/subset_result.json

@@ -0,0 +1,42 @@
+{
+  "testPaths": [
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test9" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test8" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test7" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test6" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test5" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test4" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test3" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test2" }
+    ],
+    [
+      { "type": "package", "name": "src/test/java/com/ninjinkun" },
+      { "type": "target", "name": "mylib_test1" }
+    ]
+  ],
+  "target": 0.1,
+  "session": { "id": "16" }
+}

tests/test_runners/test_bazel.py

@@ -0,0 +1,81 @@
+from pathlib import Path
+import responses
+import json
+import gzip
+import itertools
+from launchable.utils.session import read_session
+from tests.cli_test_case import CliTestCase
+
+
+class BazelTest(CliTestCase):
+    test_files_dir = Path(__file__).parent.joinpath(
+        '../data/bazel/').resolve()
+    subset_input = """Starting local Bazel server and connecting to it...
+//src/test/java/com/ninjinkun:mylib_test9
+//src/test/java/com/ninjinkun:mylib_test8
+//src/test/java/com/ninjinkun:mylib_test7
+//src/test/java/com/ninjinkun:mylib_test6
+//src/test/java/com/ninjinkun:mylib_test5
+//src/test/java/com/ninjinkun:mylib_test4
+//src/test/java/com/ninjinkun:mylib_test3
+//src/test/java/com/ninjinkun:mylib_test2
+//src/test/java/com/ninjinkun:mylib_test1
+Loading: 2 packages loaded
+"""
+
+    @responses.activate
+    def test_subset(self):
+        result = self.cli('subset', '--target', '10%',
+                          '--build', self.build_name, 'bazel', input=self.subset_input)
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(read_session(self.build_name), self.session)
+
+        payload = json.loads(gzip.decompress(
+            responses.calls[1].request.body).decode())
+        expected = self.load_json_from_file(
+            self.test_files_dir.joinpath('subset_result.json'))
+        self.assert_json_orderless_equal(payload, expected)
+
+    @responses.activate
+    def test_record_test(self):
+        result = self.cli('record', 'tests', '--build',
+                          self.build_name, 'bazel', str(self.test_files_dir) + "/")
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(read_session(self.build_name), self.session)
+
+        payload = json.loads(gzip.decompress(
+            b''.join(responses.calls[1].request.body)).decode())
+        expected = self.load_json_from_file(
+            self.test_files_dir.joinpath('record_test_result.json'))
+
+        for c in payload['events']:
+            del c['created_at']
+
+        self.assert_json_orderless_equal(payload, expected)
+
+    @responses.activate
+    def test_subset_record_key_match(self):
+        """
+        Test recorded test results contain subset's test path
+        """
+        result = self.cli('subset', '--target', '10%',
+                          '--build', self.build_name, 'bazel', input=self.subset_input)
+        self.assertEqual(result.exit_code, 0)
+
+        subset_payload = json.loads(gzip.decompress(
+            responses.calls[1].request.body).decode())
+
+        result = self.cli('record', 'tests', '--build',
+                          self.build_name, 'bazel', str(self.test_files_dir) + "/")
+        self.assertEqual(result.exit_code, 0)
+
+        record_payload = json.loads(gzip.decompress(
+            b''.join(responses.calls[2].request.body)).decode())
+
+        record_test_paths = itertools.chain.from_iterable(e['testPath'] for e in record_payload['events'])
+        record_test_path_dict = { t['name'] : t for t in record_test_paths }
+
+        for test_paths in subset_payload['testPaths']:
+            for subset_test_path in test_paths:
+                record_test_path = record_test_path_dict.get(subset_test_path['name'])
+                self.assert_json_orderless_equal(record_test_path, subset_test_path)