forked from ray-project/ray
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'ray-project:master' into master
- Loading branch information
Showing
2,445 changed files
with
186,151 additions
and
69,789 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| import argparse | ||
| import os | ||
| from collections import OrderedDict | ||
| import sys | ||
| import time | ||
| import subprocess | ||
| from typing import List | ||
|
|
||
| from aws_requests_auth.boto_utils import BotoAWSRequestsAuth | ||
| import requests | ||
|
|
||
|
|
||
| def retry(f): | ||
| def inner(): | ||
| resp = None | ||
| for _ in range(5): | ||
| resp = f() | ||
| print("Getting Presigned URL, status_code", resp.status_code) | ||
| if resp.status_code >= 500: | ||
| print("errored, retrying...") | ||
| print(resp.text) | ||
| time.sleep(5) | ||
| else: | ||
| return resp | ||
| if resp is None or resp.status_code >= 500: | ||
| print("still errorred after many retries") | ||
| sys.exit(1) | ||
|
|
||
| return inner | ||
|
|
||
|
|
||
| @retry | ||
| def perform_auth(): | ||
| auth = BotoAWSRequestsAuth( | ||
| aws_host="vop4ss7n22.execute-api.us-west-2.amazonaws.com", | ||
| aws_region="us-west-2", | ||
| aws_service="execute-api", | ||
| ) | ||
| resp = requests.get( | ||
| "https://vop4ss7n22.execute-api.us-west-2.amazonaws.com/endpoint/", | ||
| auth=auth, | ||
| params={"job_id": os.environ["BUILDKITE_JOB_ID"]}) | ||
| return resp | ||
|
|
||
|
|
||
| def handle_docker_login(resp): | ||
| pwd = resp.json()["docker_password"] | ||
| subprocess.call( | ||
| ["docker", "login", "--username", "raytravisbot", "--password", pwd]) | ||
|
|
||
|
|
||
| def gather_paths(dir_path) -> List[str]: | ||
| assert os.path.exists(dir_path) | ||
| if os.path.isdir(dir_path): | ||
| paths = [os.path.join(dir_path, f) for f in os.listdir(dir_path)] | ||
| else: | ||
| paths = [dir_path] | ||
| return paths | ||
|
|
||
|
|
||
| dest_resp_mapping = { | ||
| "wheels": "presigned_resp_prod_wheels", | ||
| "branch_wheels": "presigned_resp_prod_wheels", | ||
| "jars": "presigned_resp_prod_wheels", | ||
| "branch_jars": "presigned_resp_prod_wheels", | ||
| "logs": "presigned_logs", | ||
| } | ||
|
|
||
|
|
||
| def upload_paths(paths, resp, destination): | ||
| dest_key = dest_resp_mapping[destination] | ||
| c = resp.json()[dest_key] | ||
| of = OrderedDict(c["fields"]) | ||
|
|
||
| sha = os.environ["BUILDKITE_COMMIT"] | ||
| branch = os.environ["BUILDKITE_BRANCH"] | ||
| bk_job_id = os.environ["BUILDKITE_JOB_ID"] | ||
|
|
||
| for path in paths: | ||
| fn = os.path.split(path)[-1] | ||
| of["key"] = { | ||
| "wheels": f"latest/{fn}", | ||
| "branch_wheels": f"{branch}/{sha}/{fn}", | ||
| "jars": f"jars/latest/{fn}", | ||
| "branch_jars": f"jars/{branch}/{sha}/{fn}", | ||
| "logs": f"bazel_events/{branch}/{sha}/{bk_job_id}/{fn}" | ||
| }[destination] | ||
| of["file"] = open(path, "rb") | ||
| r = requests.post(c["url"], files=of) | ||
| print(f"Uploaded {path} to {of['key']}", r.status_code) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| parser = argparse.ArgumentParser( | ||
| description="Helper script to upload files to S3 bucket") | ||
| parser.add_argument("--path", type=str, required=False) | ||
| parser.add_argument("--destination", type=str) | ||
| args = parser.parse_args() | ||
|
|
||
| assert args.destination in { | ||
| "branch_jars", "branch_wheels", "jars", "logs", "wheels", | ||
| "docker_login" | ||
| } | ||
| assert "BUILDKITE_JOB_ID" in os.environ | ||
| assert "BUILDKITE_COMMIT" in os.environ | ||
|
|
||
| resp = perform_auth() | ||
|
|
||
| if args.destination == "docker_login": | ||
| handle_docker_login(resp) | ||
| else: | ||
| paths = gather_paths(args.path) | ||
| print("Planning to upload", paths) | ||
| upload_paths(paths, resp, args.destination) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| common: &common | ||
| artifact_paths: | ||
| - "/tmp/bazel_event_logs/*" | ||
| env: | ||
| BUILDKITE: "true" | ||
| CI: "true" | ||
| PYTHON: "3.6" | ||
| RAY_USE_RANDOM_PORTS: "1" | ||
| RAY_DEFAULT_BUILD: "1" | ||
| LC_ALL: en_US.UTF-8 | ||
| LANG: en_US.UTF-8 | ||
|
|
||
| prelude_commands: &prelude_commands |- | ||
| ./ci/travis/upload_build_info.sh | ||
| (which bazel && bazel clean) || true; | ||
| . ./ci/travis/ci.sh init && source ~/.zshrc | ||
| . ./ci/travis/ci.sh build | ||
| ./ci/travis/install-dependencies.sh | ||
|
|
||
| epilogue_commands: &epilogue_commands |- | ||
| # Cleanup runtime environment to save storage | ||
| rm -rf /tmp/ray | ||
| # Cleanup local caches (this shouldn't clean up global disk cache) | ||
| bazel clean | ||
|
|
||
| steps: | ||
| - label: ":mac: :apple: Wheels and Jars" | ||
| <<: *common | ||
| commands: | ||
| # Cleanup environments | ||
| - ./ci/travis/upload_build_info.sh | ||
| - (which bazel && bazel clean) || true | ||
| # TODO(simon): make sure to change both PR and wheel builds | ||
| # Special setup for jar builds (will be installed to the machine instead) | ||
| # - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask | ||
| # - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8 | ||
| - diskutil list external physical | ||
| - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home | ||
| - java -version | ||
| # Build wheels | ||
| - export MAC_WHEELS=1 | ||
| - export MAC_JARS=1 | ||
| - export RAY_INSTALL_JAVA=1 | ||
| - . ./ci/travis/ci.sh init && source ~/.zshrc | ||
| - . ./ci/travis/ci.sh build | ||
| # Test wheels | ||
| - . ./ci/travis/ci.sh test_wheels | ||
| # Build jars | ||
| - bash ./java/build-jar-multiplatform.sh darwin | ||
| # Upload the wheels and jars | ||
| # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated. | ||
| - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi | ||
| - pip install -q docker aws_requests_auth boto3 | ||
| # Upload to branch directory. | ||
| - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl | ||
| - python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin | ||
| # Upload to latest directory. | ||
| - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi | ||
| - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi | ||
|
|
||
|
|
||
| - label: ":mac: :apple: Ray C++ and Libraries" | ||
| <<: *common | ||
| commands: | ||
| - *prelude_commands | ||
| - TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh | ||
| - bazel test --config=ci --test_env=CI $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-flaky,-flaky-mac -- | ||
| //:all python/ray/serve/... python/ray/new_dashboard/... -rllib/... -core_worker_test | ||
| - *epilogue_commands | ||
|
|
||
| - label: ":mac: :apple: Small and Large" | ||
| <<: *common | ||
| commands: | ||
| - *prelude_commands | ||
| - bazel test $(./scripts/bazel_export_options) --config=ci | ||
| --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX | ||
| --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --test_env=CI | ||
| --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-flaky,-flaky-mac | ||
| python/ray/tests/... | ||
| - *epilogue_commands | ||
|
|
||
| - label: ":mac: :apple: Medium A-J" | ||
| <<: *common | ||
| commands: | ||
| - *prelude_commands | ||
| - bazel test --config=ci $(./scripts/bazel_export_options) --test_env=CI | ||
| --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j,-flaky,-flaky-mac | ||
| python/ray/tests/... | ||
| - *epilogue_commands | ||
|
|
||
| - label: ":mac: :apple: Medium K-Z" | ||
| <<: *common | ||
| commands: | ||
| - *prelude_commands | ||
| - bazel test --config=ci $(./scripts/bazel_export_options) --test_env=CI | ||
| --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z,-flaky,-flaky-mac | ||
| python/ray/tests/... | ||
| - *epilogue_commands | ||
|
|
||
| - label: ":mac: :apple: :snowflake: Flaky" | ||
| <<: *common | ||
| commands: | ||
| - *prelude_commands | ||
| - ./ci/travis/install-dependencies.sh | ||
| - bazel test --config=ci $(./scripts/bazel_export_options) | ||
| --test_tag_filters=-kubernetes,-jenkins_only,flaky,flaky-mac | ||
| --test_env=CONDA_EXE | ||
| --test_env=CONDA_PYTHON_EXE | ||
| --test_env=CONDA_SHLVL | ||
| --test_env=CONDA_PREFIX | ||
| --test_env=CONDA_DEFAULT_ENV | ||
| --test_env=CI | ||
| python/ray/tests/... python/ray/serve/... | ||
| - *epilogue_commands |
Oops, something went wrong.