Execute shell commands before running Bazel

buildkite/README.md

@@ -114,4 +114,39 @@ GOOD_BAZEL_COMMIT=b6ea3b6caa7f379778e74da33d1bd0ff6477f963
 BAD_BAZEL_COMMIT=91eb3d207714af0ab1e5812252a0f10f40d6e4a8
 ```
 
-Note: Bazel commit can only be set to commits after [63453bdbc6b05bd201375ee9e25b35010ae88aab](https://github.com/bazelbuild/bazel/commit/63453bdbc6b05bd201375ee9e25b35010ae88aab), Culprit Finder needs to download Bazel at specific commit, but we didn't prebuilt Bazel binaries before this commit.
+Note: Bazel commit can only be set to commits after [63453bdbc6b05bd201375ee9e25b35010ae88aab](https://github.com/bazelbuild/bazel/commit/63453bdbc6b05bd201375ee9e25b35010ae88aab), Culprit Finder needs to download Bazel at specific commit, but we didn't prebuilt Bazel binaries before this commit.
+
+## Running Buildifier on CI
+
+For each pipeline you can enable [Buildifier](https://github.com/bazelbuild/buildtools/tree/master/buildifier) to check whether all BUILD and .bzl files comply with the standard formatting convention. Simply add the following code to the top of the particular pipeline Yaml configuration (either locally in `.bazelci/presubmit.yml` or in https://github.com/bazelbuild/continuous-integration/tree/master/buildkite/pipelines):
+
+```
+buildifier: {}
+```
+
+As a consequence, every future build for this pipeline will contain an additional "Buildifier" step that runs the latest version of Buildifier on all BUILD and .bzl files in "lint" mode.
+
+### Running a specific version of Buildifier
+
+You can also specify which version of Buildifier should be run:
+
+```
+buildifier:
+  version: latest
+```
+
+The configuration value can be a hardcoded version string such as "0.20.0" or a dynamic reference such as "latest", "latest-1", etc. The latest Buildifier version will be used if the configuration does not contain the "version" field.
+
+### Specifying the set of input files
+
+By default Buildifier only processes BUILD and .bzl files. This can be changed by adding the "files" field to the configuration:
+
+```
+buildifier:
+  files:
+  - "*.bzl"
+  - "BUILD.bazel"
+  - "BUILD"
+```
+
+All values must be valid input for [find(1)](https://linux.die.net/man/1/find).

buildkite/bazelci.py

@@ -18,6 +18,7 @@
 import base64
 import codecs
 import datetime
+from distutils.version import LooseVersion
 import hashlib
 import json
 import multiprocessing
@@ -36,6 +37,7 @@
 import uuid
 import yaml
 from urllib.request import url2pathname
+from urllib.request import urlopen
 from urllib.parse import urlparse
 
 # Initialize the random number generator.
@@ -309,6 +311,16 @@
     },
 }
 
+LATEST_VERSION_PATTERN = re.compile(r"latest(-(?P<offset>\d+))?$")
+
+BUILDIFIER_RELEASE_PAGE = "https://api.github.com/repos/bazelbuild/buildtools/releases"
+
+BUILDIFIER_DEFAULT_VERSION = "latest"
+
+BUILDIFIER_DEFAULT_INPUT_FILES = ["BUILD", "*.bzl"]
+
+BUILDIFIER_DEFAULT_DOCKER_IMAGE = "gcr.io/bazel-untrusted/ubuntu1804:nojava"
+
 # The platform used for various steps (e.g. stuff that formerly ran on the "pipeline" workers).
 DEFAULT_PLATFORM = "ubuntu1804"
 
@@ -374,7 +386,7 @@ def bazelcipy_url():
     """
     URL to the latest version of this script.
     """
-    return "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py?{}".format(
+    return "https://raw.githubusercontent.com/fweikert/continuous-integration/nested2/buildkite/bazelci.py?{}".format(
         int(time.time())
     )
 
@@ -459,6 +471,11 @@ def execute_commands(
         else:
             git_repository = os.getenv("BUILDKITE_REPO")
 
+        if platform == "windows":
+            execute_batch_commands(config.get("batch_commands", None))
+        else:
+            execute_shell_commands(config.get("shell_commands", None))
+
         if use_bazel_at_commit:
             print_collapsed_group(":gcloud: Downloading Bazel built at " + use_bazel_at_commit)
             bazel_binary = download_bazel_binary_at_commit(tmpdir, platform, use_bazel_at_commit)
@@ -477,10 +494,6 @@ def execute_commands(
             for flag in incompatible_flags:
                 eprint(flag + "\n")
 
-        if platform == "windows":
-            execute_batch_commands(config.get("batch_commands", None))
-        else:
-            execute_shell_commands(config.get("shell_commands", None))
         execute_bazel_run(
             bazel_binary, platform, config.get("run_targets", None), incompatible_flags
         )
@@ -1052,7 +1065,21 @@ def execute_command_background(args):
 def create_step(label, commands, platform=DEFAULT_PLATFORM):
     host_platform = PLATFORMS[platform].get("host-platform", platform)
     if "docker-image" in PLATFORMS[platform]:
+        return create_docker_step(label, commands, PLATFORMS[platform]["docker-image"])
+    else:
         return {
+            "label": label,
+            "command": commands,
+            "agents": {
+                "kind": "worker",
+                "java": PLATFORMS[platform]["java"],
+                "os": rchop(host_platform, "_nojava", "_java8", "_java9", "_java10"),
+            },
+        }
+
+
+def create_docker_step(label, commands, docker_image):
+    return {
             "label": label,
             "command": commands,
             "agents": {"kind": "docker", "os": "linux"},
@@ -1061,7 +1088,7 @@ def create_step(label, commands, platform=DEFAULT_PLATFORM):
                     "always-pull": True,
                     "debug": True,
                     "environment": ["BUILDKITE_ARTIFACT_UPLOAD_DESTINATION", "BUILDKITE_GS_ACL"],
-                    "image": PLATFORMS[platform]["docker-image"],
+                    "image": docker_image,
                     "privileged": True,
                     "propagate-environment": True,
                     "tmpfs": ["/home/bazel/.cache:exec,uid=999,gid=999"],
@@ -1074,20 +1101,10 @@ def create_step(label, commands, platform=DEFAULT_PLATFORM):
                 }
             },
         }
-    else:
-        return {
-            "label": label,
-            "command": commands,
-            "agents": {
-                "kind": "worker",
-                "java": PLATFORMS[platform]["java"],
-                "os": rchop(host_platform, "_nojava", "_java8", "_java9", "_java10"),
-            },
-        }
 
 
 def print_project_pipeline(
-    platform_configs,
+    configs,
     project_name,
     http_config,
     file_config,
@@ -1096,6 +1113,7 @@ def print_project_pipeline(
     use_but,
     incompatible_flags,
 ):
+    platform_configs = configs.get("platforms", None)
     if not platform_configs:
         raise BuildkiteException("{0} pipeline configuration is empty.".format(project_name))
 
@@ -1122,6 +1140,10 @@ def print_project_pipeline(
         )
         pipeline_steps.append(step)
 
+    buildifier_step = get_buildifier_step_if_requested(configs)
+    if buildifier_step:
+        pipeline_steps.append(buildifier_step)
+
     pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
     all_downstream_pipeline_slugs = []
     for _, config in DOWNSTREAM_PROJECTS.items():
@@ -1198,6 +1220,71 @@ def fetch_incompatible_flag_verbose_failures_command():
     )
 
 
+def get_buildifier_step_if_requested(configs):
+    buildifier = configs.get("buildifier")
+    # An empty dictionary is allowed.
+    if buildifier is None:
+        return None
+
+    version_from_config = buildifier.get("version", BUILDIFIER_DEFAULT_VERSION)
+    version, url = get_buildifier_version_and_url(version_from_config)
+    files = buildifier.get("files", BUILDIFIER_DEFAULT_INPUT_FILES)
+    return create_buildifier_step(version, url, files)
+
+
+def get_buildifier_version_and_url(version_from_config):
+    releases = get_all_buildifier_releases()
+
+    requested_release = None
+    if "latest" in version_from_config:
+        requested_release = get_latest_buildifier_release(releases.values(), version_from_config)
+    else:
+        requested_release = releases.get(version_from_config)
+        if not requested_release:
+            raise BuildkiteException("There is no Buildifier version '{}'.".format(version_from_config))
+
+    urls = [a["browser_download_url"] for a in requested_release["assets"] if a["name"] == "buildifier"]
+    if not urls:
+        raise BuildkiteException("There is no download URL for Buildifier release '{}'.".format(version_from_config))
+
+    return requested_release["tag_name"], urls[0]
+
+
+def get_all_buildifier_releases():
+    res = urlopen("{}?{}".format(BUILDIFIER_RELEASE_PAGE, int(time.time()))).read()
+    return {r["tag_name"] : r for r in json.loads(res.decode('utf-8')) if not r['prerelease']}
+
+
+def get_latest_buildifier_release(releases, version_from_config):
+    match = LATEST_VERSION_PATTERN.match(version_from_config)
+    if not match:
+        raise BuildkiteException("Invalid version '{}'. In addition to using a version "
+                                 "number such as '0.20.0', you can use values such as "
+                                 "latest' and 'latest-N', with N being a non-negative "
+                                 "integer.".format(version_from_config))
+
+    offset = int(match.group("offset") or "0")
+    sorted_releases = sorted(releases, reverse=True, key= lambda r: LooseVersion(r["tag_name"]))
+    if offset >= len(sorted_releases):
+        version = "latest-{}".format(offset) if offset else "latest"
+        raise BuildkiteException("Cannot resolve version '{}': There are only {} Buildifier "
+                                 "releases.".format(version, len(sorted_releases))) 
+
+    return sorted_releases[offset]
+
+
+def create_buildifier_step(version, url, files, os="ubuntu1604"):
+    commands = [            "curl -L {} -o buildifier".format(url),
+            "chmod +x buildifier",
+            create_buildifier_command(files)        ]
+    return create_docker_step("Buildifier {}".format(version), commands, BUILDIFIER_DEFAULT_DOCKER_IMAGE)
+
+
+def create_buildifier_command(files_to_lint):
+    find_args = " -or ".join('-iname "{}"'.format(f) for f in files_to_lint)
+    return "./buildifier --lint=warn $$(find . -type f \\( {} \\)) | grep -q \".\"".format(find_args) 
+
+
 def upload_project_pipeline_step(
     project_name, git_repository, http_config, file_config, incompatible_flags
 ):
@@ -1745,7 +1832,7 @@ def main(argv=None):
         elif args.subparsers_name == "project_pipeline":
             configs = fetch_configs(args.http_config, args.file_config)
             print_project_pipeline(
-                platform_configs=configs.get("platforms", None),
+                configs=configs,
                 project_name=args.project_name,
                 http_config=args.http_config,
                 file_config=args.file_config,