Bump version to 1.22.0

awslabs · Mar 15, 2022 · 9ce9c61 · 9ce9c61
2 parents 8768bb6 + a3bebfb
commit 9ce9c61
Show file tree

Hide file tree

Showing 9 changed files with 161 additions and 111 deletions.
diff --git a/.github/workflows/run-tests → .github/workflows/run-tests.yaml b/.github/workflows/run-tests → .github/workflows/run-tests.yaml
@@ -1,11 +1,13 @@
 name: Run Litani Tests
 on:
   pull_request:
-    types: [ labeled ]
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+  push:
+    branches: [ release, develop ]
 
 jobs:
   test-litani:
-    if: ${{ github.event.label.name == 'test' }}
+    if: "!contains(github.event.pull_request.labels.*.name, 'no-test')"
     name: Run Litani Tests
     runs-on: macos-latest
     steps:

diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,47 @@
                               CHANGELOG
                               `````````
 
+Version 1.22.0 -- 2022-03-15
+----------------------------
+- Print out stderr when a job fails
+  The entire buffered stderr of a job that fails will now be printed to
+  the terminal after the failing command line. This is to help users
+  quickly debug these jobs without viewing the HTML report.
+
+  This commit fixes #131.
+
+- Fix content colour in dark mode
+
+  Previously, some content would appear in a dark colour in dark mode
+  because the "color" property was set in the .content class for light
+  mode but the <body> element for dark mode.
+
+- Rebuild run graph in transform jobs
+
+  After receiving user input in transform jobs, discard old jobs and make
+  a new run graph using the jobs received on stdin. This makes it so that
+  running add-job in parallel with transform-jobs will fail.
+
+- Add ids to sections on HTML dashboard
+
+  Every major section on the HTML dashboard now has an "id" attribute,
+  making it possible to link to those sections. Prior to this commit, it
+  was not possible to link to specific graphs on the front page, for
+  example. This PR also introduces a CONTRIBUTING.md file that contains
+  guidance to continue this pattern.
+
+- Do not run litani tests if 'no-test' label is set
+
+- Fix space in release script
+
+- Tell release engineer to push develop and release
+
+  Previously, following the instructions would only push the release
+  branch to origin, not the develop branch.
+
+- Fix run-tests workflow file name
+
+
 Version 1.21.0 -- 2022-03-04
 ----------------------------
 - Add release script

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,16 @@
+Contributing
+============
+
+Thank you for contributing to Litani! This document collects some coding and
+process guidelines.
+
+
+### HTML Dashboard
+
+- Please test your changes with both light and dark mode, and with a range of
+  browser widths.
+- Almost all top-level divs should have an id attribute; this makes it easy to
+  link to specific information.
+- We prefer to inline all assets (CSS, images) onto the page so that it's easy
+  to send single, self-contained pages around. For this reason, please try to
+  keep SVGs small.
diff --git a/lib/add_job.py b/lib/add_job.py
@@ -0,0 +1,55 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License
+
+import json
+import logging
+import sys
+import uuid
+
+from lib import litani
+
+
+async def add_job(job_dict):
+    cache_file = litani.get_cache_dir() / litani.CACHE_FILE
+    with open(cache_file) as handle:
+        cache_contents = json.load(handle)
+    if job_dict["ci_stage"] not in cache_contents["stages"]:
+        valid_stages = "', '".join(cache_contents["stages"])
+        logging.error(
+            "Invalid stage name '%s' was provided, possible "
+            "stage names are: '%s'", job_dict["ci_stage"], valid_stages)
+        sys.exit(1)
+
+    jobs_dir = litani.get_cache_dir() / litani.JOBS_DIR
+    jobs_dir.mkdir(exist_ok=True, parents=True)
+
+    if job_dict["phony_outputs"]:
+        if not job_dict["outputs"]:
+            job_dict["outputs"] = job_dict["phony_outputs"]
+        else:
+            for phony_output in job_dict["phony_outputs"]:
+                if phony_output not in job_dict["outputs"]:
+                    job_dict["outputs"].append(phony_output)
+
+    if "func" in job_dict:
+        job_dict.pop("func")
+
+    job_id = str(uuid.uuid4())
+    job_dict["job_id"] = job_id
+    job_dict["status_file"] = str(
+        litani.get_status_dir() / ("%s.json" % job_id))
+
+    logging.debug("Adding job: %s", json.dumps(job_dict, indent=2))
+
+    with litani.atomic_write(jobs_dir / ("%s.json" % job_id)) as handle:
+        print(json.dumps(job_dict, indent=2), file=handle)
diff --git a/lib/litani.py b/lib/litani.py
@@ -33,7 +33,7 @@
 TIME_FORMAT_W = "%Y-%m-%dT%H:%M:%SZ"
 TIME_FORMAT_MS = "%Y-%m-%dT%H:%M:%S.%fZ"
 VERSION_MAJOR = 1
-VERSION_MINOR = 21
+VERSION_MINOR = 22
 VERSION_PATCH = 0
 RC = False
 

diff --git a/lib/transform_jobs.py b/lib/transform_jobs.py
@@ -20,6 +20,7 @@
 import sys
 
 import lib.litani
+import lib.add_job
 
 
 @dataclasses.dataclass
@@ -28,76 +29,35 @@ class _JobsTransformer:
     old_uuids: list
 
 
-    def __call__(self, user_jobs):
+    async def __call__(self, user_jobs):
+        self._delete_old_jobs()
+        await self._add_new_jobs(user_jobs)
+
+
+    async def _add_new_jobs(self, user_jobs):
+        for job in user_jobs:
+            job["subcommand"] = "add-job"
+            await lib.add_job.add_job(job)
+
+
+    def _delete_old_jobs(self):
         for uuid in self.old_uuids:
             job_file = self.jobs_dir / ("%s.json" % uuid)
             try:
-                with open(job_file) as handle:
-                    old_job = json.load(handle)
-                    uuid = old_job["job_id"]
+                os.unlink(job_file)
             except FileNotFoundError:
-                logging.warning(
-                    "file for job %s disappeared; not updating", uuid)
                 continue
 
-            updated_job = [
-                j for j in user_jobs if j["job_id"] == uuid
-            ]
-            if not updated_job:
-                self._delete_job(old_job)
-            elif len(updated_job) > 1:
-                logging.error(
-                    "user input contained two jobs with uuid %s", uuid)
-                sys.exit(1)
-            elif updated_job[0] == old_job:
-                self._write_unmodified_job(old_job)
-            else:
-                self._write_transformed_job(updated_job[0])
-
-        new_jobs = [
-            j for j in user_jobs
-            if j["job_id"] not in self.old_uuids]
-        for job in new_jobs:
-            self._write_new_job(job)
-
-
-    @staticmethod
-    def _job_name(job):
-        desc = f" ({job['description']})" if job["description"] else ''
-        return f"{job['job_id']}{desc}"
-
-
-    def _path_to_job(self, job):
-        return self.jobs_dir / ("%s.json" % job["job_id"])
-
-
-    def _write_new_job(self, job):
-        logging.info("writing new job %s", self._job_name(job))
-        with lib.litani.atomic_write(self._path_to_job(job)) as handle:
-            print(json.dumps(job, indent=2), file=handle)
-
-
-    def _write_unmodified_job(self, job):
-        logging.debug("not changing job %s", self._job_name(job))
-
-
-    def _write_transformed_job(self, job):
-        logging.info("transforming job %s", self._job_name(job))
-        with lib.litani.atomic_write(self._path_to_job(job)) as handle:
-            print(json.dumps(job, indent=2), file=handle)
-
-
-    def _delete_job(self, job):
-        logging.info("deleting job %s", self._job_name(job))
-        os.unlink(self._path_to_job(job))
-
 
 
 def _print_jobs(job_paths):
     out = []
     for job in job_paths:
         with open(job) as handle:
-            out.append(json.load(handle))
+            job_dict = json.load(handle)
+            for key in ("job_id", "status_file", "subcommand"):
+                job_dict.pop(key)
+            out.append(job_dict)
     print(json.dumps(out, indent=2))
     sys.stdout.flush()
     os.close(sys.stdout.fileno())
@@ -121,4 +81,4 @@ async def main(_):
     new_jobs = _read_jobs()
 
     transform = _JobsTransformer(jobs_dir, old_uuids)
-    transform(new_jobs)
+    await transform(new_jobs)
diff --git a/litani b/litani
@@ -32,6 +32,7 @@ import traceback
 import uuid
 
 from lib import litani, litani_report, ninja_syntax
+import lib.add_job
 import lib.capabilities
 import lib.graph
 import lib.job_outcome
@@ -641,39 +642,7 @@ async def init(args):
 
 
 async def add_job(args):
-    cache_file = litani.get_cache_dir() / litani.CACHE_FILE
-    with open(cache_file) as handle:
-        cache_contents = json.load(handle)
-    if args.ci_stage not in cache_contents["stages"]:
-        valid_stages = "', '".join(cache_contents["stages"])
-        logging.error(
-            "Invalid stage name '%s' was provided, possible "
-            "stage names are: '%s'", args.ci_stage, valid_stages)
-        sys.exit(1)
-
-    jobs_dir = litani.get_cache_dir() / litani.JOBS_DIR
-    jobs_dir.mkdir(exist_ok=True, parents=True)
-
-    if args.phony_outputs:
-        if not args.outputs:
-            args.outputs = args.phony_outputs
-        else:
-            for phony_output in args.phony_outputs:
-                if phony_output not in args.outputs:
-                    args.outputs.append(phony_output)
-
-    job = vars(args)
-    job.pop("func")
-
-    job_id = str(uuid.uuid4())
-    job["job_id"] = job_id
-    job["status_file"] = str(
-        litani.get_status_dir() / ("%s.json" % job_id))
-
-    logging.debug("Adding job: %s", json.dumps(job, indent=2))
-
-    with litani.atomic_write(jobs_dir / ("%s.json" % job_id)) as handle:
-        print(json.dumps(job, indent=2), file=handle)
+    await lib.add_job.add_job(vars(args))
 
 
 async def run_build(args):
@@ -803,6 +772,11 @@ async def exec_job(args):
             with litani.atomic_write(arg_file) as handle:
                 print(out_str, file=handle)
 
+    if out_data["wrapper_return_code"] and out_data["stderr"]:
+        print(
+            "\n".join([l.rstrip() for l in out_data["stderr"]]),
+            file=sys.stderr)
+
     timestamp("end_time", out_data)
     out_str = json.dumps(out_data, indent=2)
     logging.debug("run status: %s", out_str)

diff --git a/script/release b/script/release
@@ -62,8 +62,11 @@ create-release: $(SANITY_CHECKS) $(STATE_FILES) $(RELEASE_PROCESS)
 	$(info --- Finished)
 	git checkout release
 	@>&2 printf '\n```````````````````````````````````````````````````````````\n'
-	@>&2 printf \
-	  "Now run\n\n  git push -u origin release\n  git push origin --tags\n\n"
+	@>&2 printf "Now run\n\n"
+	@>&2 printf "  git push -u origin release\n"
+	@>&2 printf "  git checkout develop\n"
+	@>&2 printf "  git push -u origin develop\n"
+	@>&2 printf "  git push origin --tags\n\n"
 	@>&2 printf '___________________________________________________________\n'
 
 .PHONY: $(SANITY_CHECKS) $(PHONY_DEPS)
@@ -98,7 +101,7 @@ create-rc: $(TMP)/rc-number
 	git add lib/litani.py
 	major=$$(grep -e '^VERSION_MAJOR' lib/litani.py | head -n 1 | awk -F= '{print $$2}'); \
 	minor=$$(cat $(TMP)/rc-number); \
-	  git commit -m "Create version $${major}.$${minor} release candidate"
+	  git commit -m "Create version$${major}.$${minor} release candidate"
 
 $(TMP)/rc-number:
 	$(info --- Generating RC number)