From 0dad4e6a2592e9ae5af449a0d30b35ef9bc5f65c Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sun, 20 Jan 2019 12:16:47 -0800 Subject: [PATCH] Build Raylet with Bazel (#3806) --- .travis.yml | 5 + .travis/install-bazel.sh | 23 +++ BUILD.bazel | 306 +++++++++++++++++++++++++++++++++++++++ WORKSPACE | 29 ++++ bazel/BUILD | 0 bazel/BUILD.plasma | 82 +++++++++++ 6 files changed, 445 insertions(+) create mode 100755 .travis/install-bazel.sh create mode 100644 BUILD.bazel create mode 100644 WORKSPACE create mode 100644 bazel/BUILD create mode 100644 bazel/BUILD.plasma diff --git a/.travis.yml b/.travis.yml index 59e5f408f78c..95b07776cc10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,11 @@ matrix: script: - ./java/test.sh + # Test Bazel build + - rm -rf build + - ./.travis/install-bazel.sh + - bazel build ... + - os: linux dist: trusty env: LINT=1 PYTHONWARNINGS=ignore diff --git a/.travis/install-bazel.sh b/.travis/install-bazel.sh new file mode 100755 index 000000000000..c9614f7722ef --- /dev/null +++ b/.travis/install-bazel.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Cause the script to exit if a single command fails +set -e + +platform="unknown" +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + echo "Platform is linux." + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + echo "Platform is macosx." + platform="darwin" +else + echo "Unrecognized platform." + exit 1 +fi + +URL="https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel-0.21.0-installer-${platform}-x86_64.sh" +wget -O install.sh $URL +chmod +x install.sh +./install.sh --user +rm -f install.sh diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 000000000000..2913cdd3ceac --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,306 @@ +# Bazel build +# C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html + +load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") + +cc_binary( + name = "raylet", + srcs = ["src/ray/raylet/main.cc"], + deps = [ + ":ray_util", + ":raylet_lib", + ], +) + +cc_binary( + name = "raylet_monitor", + srcs = [ + "src/ray/raylet/monitor.cc", + "src/ray/raylet/monitor.h", + "src/ray/raylet/monitor_main.cc", + ], + deps = [ + ":gcs", + ":ray_util", + ], +) + +cc_library( + name = "raylet_lib", + srcs = glob([ + "src/ray/raylet/*.cc" + ], exclude = [ + "src/ray/raylet/mock_gcs_client.cc", + "src/ray/raylet/monitor_main.cc", + "src/ray/raylet/*_test.cc", + ]), + hdrs = glob([ + "src/ray/raylet/*.h", + ]), + deps = [ + ":gcs", + ":gcs_fbs", + ":node_manager_fbs", + ":object_manager", + ":ray_common", + ":ray_util", + "@boost//:asio", + "@plasma", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "lineage_cache_test", + srcs = ["src/ray/raylet/lineage_cache_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":node_manager_fbs", + ":raylet_lib", + ], +) + +cc_test( + name = "reconstruction_policy_test", + srcs = ["src/ray/raylet/reconstruction_policy_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":node_manager_fbs", + ":object_manager", + ":raylet_lib" + ], +) + +cc_test( + name = "worker_pool_test", + srcs = ["src/ray/raylet/worker_pool_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":raylet_lib", + ], +) + +cc_test( + name = "logging_test", + srcs = ["src/ray/util/logging_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":ray_util", + ], +) + +cc_test( + name = "task_dependency_manager_test", + srcs = ["src/ray/raylet/task_dependency_manager_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":raylet_lib" + ], +) + +cc_test( + name = "task_test", + srcs = ["src/ray/raylet/task_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":raylet_lib", + ], +) + +cc_library( + name = "object_manager", + srcs = glob([ + "src/ray/object_manager/*.cc", + ]), + hdrs = glob([ + "src/ray/object_manager/*.h", + ]), + includes = [ + "src", + ], + deps = [ + ":gcs", + ":object_manager_fbs", + ":ray_common", + ":ray_util", + "@boost//:asio", + "@plasma" + ], +) + +cc_binary( + name = "object_manager_test", + testonly = 1, + srcs = ["src/ray/object_manager/test/object_manager_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":object_manager", + ], +) + +cc_binary( + name = "object_manager_stress_test", + testonly = 1, + srcs = ["src/ray/object_manager/test/object_manager_stress_test.cc"], + deps = [ + "@com_google_googletest//:gtest_main", + ":object_manager", + ], +) + +cc_library( + name = "ray_util", + srcs = glob([ + "src/ray/*.cc", + "src/ray/util/*.cc", + ], exclude = [ + "src/ray/util/logging_test.cc", + "src/ray/util/signal_test.cc" + ]), + hdrs = glob([ + "src/ray/*.h", + "src/ray/util/*.h", + ]), + includes = [ + "src", + ], + deps = [ + "@plasma", + ":sha256" + ] +) + +cc_library( + name = "ray_common", + srcs = [ + "src/ray/common/client_connection.cc", + "src/ray/common/common_protocol.cc" + ], + hdrs = [ + "src/ray/common/client_connection.h", + "src/ray/common/common_protocol.h" + ], + includes = [ + "src/ray/gcs/format", + ], + deps = [ + ":gcs_fbs", + ":node_manager_fbs", + ":ray_util", + "@boost//:asio", + ], +) + +cc_library( + name = "sha256", + srcs = [ + "src/ray/thirdparty/sha256.c", + ], + hdrs = [ + "src/ray/thirdparty/sha256.h", + ], + includes = ["src/ray/thirdparty"], +) + +cc_library( + name = "hiredis", + srcs = glob([ + "src/ray/thirdparty/ae/ae.c", + "src/ray/thirdparty/hiredis/*.c", + ], exclude = [ + "src/ray/thirdparty/hiredis/test.c" + ]), + hdrs = glob([ + "src/ray/thirdparty/ae/*.h", + "src/ray/thirdparty/hiredis/*.h", + "src/ray/thirdparty/hiredis/adapters/*.h", + "src/ray/thirdparty/hiredis/dict.c", + "src/ray/thirdparty/ae/ae_kqueue.c", + "src/ray/thirdparty/ae/ae_epoll.c", + ]), + includes = [ + "src/ray/thirdparty/hiredis", + "src/ray/thirdparty/ae", + ], +) + +cc_library( + name = "gcs", + srcs = glob([ + "src/ray/gcs/*.cc" + ], exclude = [ + "src/ray/gcs/*_test.cc" + ]), + hdrs = glob([ + "src/ray/gcs/*.h", + "src/ray/gcs/format/*.h", + ]), + includes = [ + "src/ray/gcs/format", + ], + deps = [ + ":gcs_fbs", + ":node_manager_fbs", + ":ray_util", + ":ray_common", + ":hiredis", + "@boost//:asio", + ], +) + +cc_binary( + name = "gcs_client_test", + testonly = 1, + srcs = ["src/ray/gcs/client_test.cc"], + deps = [ + ":gcs", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "asio_test", + testonly = 1, + srcs = ["src/ray/gcs/asio_test.cc"], + deps = [ + ":gcs", + "@com_google_googletest//:gtest_main", + ":ray_util", + ], +) + +FLATC_ARGS = [ + "--gen-object-api", + "--gen-mutable", + "--scoped-enums", +] + +flatbuffer_cc_library( + name = "gcs_fbs", + srcs = ["src/ray/gcs/format/gcs.fbs"], + flatc_args = FLATC_ARGS, + out_prefix = "src/ray/gcs/format/", +) + +flatbuffer_cc_library( + name = "common_fbs", + srcs = ["@plasma//:cpp/src/plasma/format/common.fbs"], + flatc_args = FLATC_ARGS, + out_prefix = "src/ray/common/" +) + +flatbuffer_cc_library( + name = "node_manager_fbs", + srcs = ["src/ray/raylet/format/node_manager.fbs"], + flatc_args = FLATC_ARGS, + include_paths = ["src/ray/gcs/format"], + includes = [":gcs_fbs_includes"], + out_prefix = "src/ray/raylet/format/", +) + +flatbuffer_cc_library( + name = "object_manager_fbs", + srcs = ["src/ray/object_manager/format/object_manager.fbs"], + flatc_args = FLATC_ARGS, + out_prefix = "src/ray/object_manager/format/", +) diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 000000000000..c44b45e89d0f --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,29 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") + +git_repository( + name = "com_github_nelhage_rules_boost", + commit = "6d6fd834281cb8f8e758dd9ad76df86304bf1869", + remote = "https://github.com/nelhage/rules_boost", +) + +load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps") +boost_deps() + +git_repository( + name = "com_github_google_flatbuffers", + remote = "https://github.com/google/flatbuffers.git", + commit = "63d51afd1196336a7d1f56a988091ef05deb1c62", +) + +git_repository( + name = "com_google_googletest", + remote = "https://github.com/google/googletest", + commit = "3306848f697568aacf4bcca330f6bdd5ce671899", +) + +new_git_repository( + name = "plasma", + build_file = "@//bazel:BUILD.plasma", + remote = "https://github.com/ray-project/arrow", + commit = "1e4f867eb1dc31107331ab1defdffb94467f31dc", +) diff --git a/bazel/BUILD b/bazel/BUILD new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma new file mode 100644 index 000000000000..23bbf3b11f28 --- /dev/null +++ b/bazel/BUILD.plasma @@ -0,0 +1,82 @@ +load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") + +cc_library( + name = "plasma", + visibility = ["//visibility:public"], + hdrs = [ + "cpp/src/plasma/client.h", + "cpp/src/plasma/common.h", + "cpp/src/plasma/compat.h", + "cpp/src/plasma/events.h", + "cpp/src/arrow/buffer.h", + "cpp/src/arrow/memory_pool.h", + "cpp/src/arrow/status.h", + "cpp/src/arrow/util/macros.h", + "cpp/src/arrow/util/visibility.h", + "cpp/src/arrow/util/string_builder.h", + # private headers + "cpp/src/plasma/common_generated.h", + "cpp/src/plasma/plasma_generated.h", + "cpp/src/plasma/eviction_policy.h", + "cpp/src/plasma/fling.h", + "cpp/src/plasma/io.h", + "cpp/src/plasma/malloc.h", + "cpp/src/plasma/plasma.h", + "cpp/src/plasma/protocol.h", + "cpp/src/plasma/store.h", + "cpp/src/plasma/thirdparty/dlmalloc.c", + "cpp/src/plasma/thirdparty/ae/ae.h", + "cpp/src/plasma/thirdparty/ae/ae_epoll.c", + "cpp/src/plasma/thirdparty/ae/ae_evport.c", + "cpp/src/plasma/thirdparty/ae/ae_kqueue.c", + "cpp/src/plasma/thirdparty/ae/ae_select.c", + "cpp/src/plasma/thirdparty/ae/config.h", + "cpp/src/plasma/thirdparty/ae/zmalloc.h", + "cpp/src/arrow/io/interfaces.h", + "cpp/src/arrow/util/bit-util.h", + "cpp/src/arrow/util/io-util.h", + "cpp/src/arrow/util/logging.h", + "cpp/src/arrow/util/string_view.h", + "cpp/src/arrow/util/type_traits.h", + "cpp/src/arrow/vendored/string_view.hpp", + "cpp/src/arrow/util/thread-pool.h", + "cpp/src/arrow/vendored/xxhash/xxhash.h", + "cpp/src/arrow/vendored/xxhash/xxhash.c", + "cpp/src/arrow/util/windows_compatibility.h", + ], + srcs = glob([ + "cpp/src/plasma/*.cc", + "cpp/src/plasma/thirdparty/ae/ae.c", + "cpp/src/arrow/buffer.cc", + "cpp/src/arrow/memory_pool.cc", + "cpp/src/arrow/status.cc", + "cpp/src/arrow/util/logging.cc", + "cpp/src/arrow/util/thread-pool.cc", + "cpp/src/arrow/util/io-util.cc", + ]), + deps = [":common_fbs", ":plasma_fbs"], + strip_include_prefix = "cpp/src", +) + +FLATC_ARGS = [ + "--gen-object-api", + "--gen-mutable", + "--scoped-enums", +] + +flatbuffer_cc_library( + name="common_fbs", + srcs=["cpp/src/plasma/format/common.fbs"], + flatc_args=FLATC_ARGS, + out_prefix="cpp/src/plasma/" +) + +flatbuffer_cc_library( + name="plasma_fbs", + srcs=["cpp/src/plasma/format/plasma.fbs"], + flatc_args=FLATC_ARGS, + out_prefix="cpp/src/plasma/", + includes = ["cpp/src/plasma/format/common.fbs"] +) + +exports_files(["cpp/src/plasma/format/common.fbs"])