diff --git a/docs/internal-documentation/service.md b/docs/internal-documentation/service.md deleted file mode 100644 index a2dc23af3..000000000 --- a/docs/internal-documentation/service.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -layout: default -title: The FuzzBench Service -parent: Internal Documentation -nav_order: 1 -permalink: /internal-documentation/service/ ---- - -# The FuzzBench service -{: .no_toc} - -**Note:** This document and most of the `service/` directory is only intended -for use by FuzzBench maintainers. It will contain hardcoded values and -references to things that don't make sense for other users. - -- TOC -{:toc} - -## Overview - -This document discusses the FuzzBench service. The service works as follows: -When a user wants a new experiment they add the experiment to -`experiment-requests.yaml`. Twice a day at 6 AM PT (13:00 UTC) and 6 PM PT -(01:00 UTC) a cron job on the `service` instance will execute the script -`run.bash`. `run.bash` will clone FuzzBench and then execute -`automatic_run_experiment.py` which starts newly requested experiments. - -## Setting up an instance to run an experiment - -This shouldn't be necessary, but here are instructions in case the current -instance is lost. -1. Run `setup.bash`. This will build and install a supported Python version, - download the `cloud_sql_proxy` and run it so that we have a connection to the - db. - -1. Install the cron job. An example you can use is in the - [crontab file](https://github.com/google/fuzzbench/tree/master/service/crontab). - Note that you must fill in `POSTGRES_PASSWORD` and `$HOME`. - -1. Verify that the service is running. One way you can debug this is by looking - at the stdout/stderr of `run.bash` which is saved in - `/tmp/fuzzbench-service.log`. If something isn't working you should probably - verify that `run.bash` works on its own. Note that `run.bash` is executed - from a checkout of FuzzBench that isn't automatically updated. So if you need - to update you must do so with `git pull --rebase`. - -## Automatic merging - -Experiments that are run using the service will be marked as nonprivate and on -completion automatically merge using clobbering. diff --git a/presubmit.py b/presubmit.py index 540173ac2..40c435ee9 100755 --- a/presubmit.py +++ b/presubmit.py @@ -31,14 +31,10 @@ import sys from typing import List, Optional -import yaml - from common import benchmark_utils from common import fuzzer_utils from common import filesystem from common import logs -from common import yaml_utils -from service import automatic_run_experiment from src_analysis import change_utils from src_analysis import diff_utils @@ -258,31 +254,6 @@ def yapf(paths: List[Path], validate: bool = True) -> bool: return success -def validate_experiment_requests(paths: List[Path]): - """Returns False if service/experiment-requests.yaml it is in |paths| and is - not valid.""" - if Path(automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH) not in paths: - return True - - try: - experiment_requests = yaml_utils.read( - automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH) - except yaml.parser.ParserError: - print('Error parsing ' - f'{automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH}.') - return False - - # Only validate the latest request. - result = automatic_run_experiment.validate_experiment_requests( - experiment_requests[:1]) - - if not result: - print(f'{automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH}' - 'is not valid.') - - return result - - def is_path_ignored(path: Path) -> bool: """Returns True if |path| is a subpath of an ignored directory or is a third_party directory.""" @@ -441,7 +412,6 @@ def main() -> int: ('typecheck', pytype), ('test', pytest), ('validate_fuzzers_and_benchmarks', validate_fuzzers_and_benchmarks), - ('validate_experiment_requests', validate_experiment_requests), ('test_changed_integrations', test_changed_integrations), ] diff --git a/service/automatic_run_experiment.py b/service/automatic_run_experiment.py deleted file mode 100644 index f1d9f180e..000000000 --- a/service/automatic_run_experiment.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Reads experiment-requests.yaml and determines if there is a new experiment -and runs it if needed. Note that this code uses a config file for experiments -that is specific to the FuzzBench service. Therefore this code will break if -others try to run it.""" -import argparse -import collections -import os -import re -import sys -from typing import Optional - -from common import benchmark_utils -from common import logs -from common import utils -from common import yaml_utils -from database import models -from database import utils as db_utils -from experiment import run_experiment - -logger = logs.Logger() # pylint: disable=invalid-name - -EXPERIMENT_CONFIG_FILE = os.path.join(utils.ROOT_DIR, 'service', - 'experiment-config.yaml') - -REQUESTED_EXPERIMENTS_PATH = os.path.join(utils.ROOT_DIR, 'service', - 'experiment-requests.yaml') - -# Don't run an experiment if we have a "request" just containing this keyword. -# TODO(metzman): Look into replacing this mechanism for pausing the service. -PAUSE_SERVICE_KEYWORD = 'PAUSE_SERVICE' - -EXPERIMENT_NAME_REGEX = re.compile(r'^\d{4}-\d{2}-\d{2}.*') -CONCURRENT_BUILDS = 30 - - -def _get_experiment_name(experiment_config: dict) -> str: - """Returns the name of the experiment described by |experiment_config| as a - string.""" - # Use str because the yaml parser will parse things like `2020-05-06` as - # a datetime if not included in quotes. - return str(experiment_config['experiment']) - - -def _get_description(experiment_config: dict) -> Optional[str]: - """Returns the description of the experiment described by - |experiment_config| as a string.""" - return experiment_config.get('description') - - -def _use_oss_fuzz_corpus(experiment_config: dict) -> bool: - """Returns the oss_fuzz_corpus flag of the experiment described by - |experiment_config| as a bool.""" - return bool(experiment_config.get('oss_fuzz_corpus')) - - -def _get_requested_experiments(): - """Return requested experiments.""" - return yaml_utils.read(REQUESTED_EXPERIMENTS_PATH) - - -def validate_experiment_name(experiment_name): - """Returns True if |experiment_name| is valid.""" - if EXPERIMENT_NAME_REGEX.match(experiment_name) is None: - logger.error('Experiment name: %s is not valid.', experiment_name) - return False - try: - run_experiment.validate_experiment_name(experiment_name) - return True - except run_experiment.ValidationError: - logger.error('Experiment name: %s is not valid.', experiment_name) - return False - - -def _validate_individual_experiment_requests(experiment_requests): - """Returns True if all requests in |experiment_request| are valid in - isolation. Does not account for PAUSE_SERVICE_KEYWORD or duplicates.""" - valid = True - # Validate format. - for request in experiment_requests: - if not isinstance(request, dict): - logger.error('Request: %s is not a dict.', request) - experiment_requests.remove(request) - valid = False - continue - - if 'experiment' not in request: - logger.error('Request: %s does not have field "experiment".', - request) - valid = False - continue - - experiment = _get_experiment_name(request) - if not validate_experiment_name(experiment): - valid = False - # Request isn't so malformed that we can find other issues, if - # present. - - fuzzers = request.get('fuzzers') - if not fuzzers: - logger.error('Request: %s does not specify any fuzzers.', request) - valid = False - continue - - for fuzzer in fuzzers: - try: - run_experiment.validate_fuzzer(fuzzer) - except run_experiment.ValidationError: - logger.error('Fuzzer: %s is invalid.', fuzzer) - valid = False - - description = request.get('description') - if description is not None and not isinstance(description, str): - logger.error( - 'Request: %s "description" attribute is not a valid string.', - request) - valid = False - - oss_fuzz_corpus = request.get('oss_fuzz_corpus') - if oss_fuzz_corpus is not None and not isinstance( - oss_fuzz_corpus, bool): - logger.error( - 'Request: %s "oss_fuzz_corpus" attribute is not a valid bool.', - request) - valid = False - - experiment_type = request.get('type', - benchmark_utils.BenchmarkType.CODE.value) - if experiment_type not in benchmark_utils.BENCHMARK_TYPE_STRS: - logger.error('Type: %s is invalid, must be one of %s', - experiment_type, benchmark_utils.BENCHMARK_TYPE_STRS) - valid = False - - benchmarks = sorted(request.get('benchmarks', [])) # Sort for testing. - for benchmark in benchmarks: - benchmark_type = benchmark_utils.get_type(benchmark) - if (benchmark_type == benchmark_utils.BenchmarkType.BUG.value and - experiment_type != benchmark_utils.BenchmarkType.BUG.value): - logger.error( - 'Benchmark %s is "type: bug". ' - 'Experiment %s must be "type: bug" as well.', benchmark, - experiment) - valid = False - break - - return valid - - -def validate_experiment_requests(experiment_requests): - """Returns True if all requests in |experiment_requests| are valid.""" - # This function tries to find as many requests as possible. - if PAUSE_SERVICE_KEYWORD in experiment_requests: - # This is a special case where a string is used instead of an experiment - # to tell the service not to run experiments automatically. Remove it - # from the list because it fails validation. - experiment_requests = experiment_requests[:] # Don't mutate input. - experiment_requests.remove(PAUSE_SERVICE_KEYWORD) - - if not _validate_individual_experiment_requests(experiment_requests): - # Don't try the next validation step if the previous failed, we might - # exception. - return False - - # Make sure experiment requests have a unique name, we can't run the same - # experiment twice. - counts = collections.Counter( - [request['experiment'] for request in experiment_requests]) - - valid = True - for experiment_name, count in counts.items(): - if count != 1: - logger.error('Experiment: "%s" appears %d times.', - str(experiment_name), count) - valid = False - - return valid - - -def run_requested_experiment(dry_run): - """Run the oldest requested experiment that hasn't been run yet in - experiment-requests.yaml.""" - requested_experiments = _get_requested_experiments() - - # TODO(metzman): Look into supporting benchmarks as an optional parameter so - # that people can add fuzzers that don't support everything. - - if PAUSE_SERVICE_KEYWORD in requested_experiments: - # Check if automated experiment service is paused. - logs.warning('Pause service requested, not running experiment.') - return - - requested_experiment = None - for experiment_config in reversed(requested_experiments): - experiment_name = _get_experiment_name(experiment_config) - with db_utils.session_scope() as session: - is_new_experiment = session.query(models.Experiment).filter( - models.Experiment.name == experiment_name).first() is None - if is_new_experiment: - requested_experiment = experiment_config - break - - if requested_experiment is None: - logs.info('No new experiment to run. Exiting.') - return - - experiment_name = _get_experiment_name(requested_experiment) - if not validate_experiment_requests([requested_experiment]): - logs.error('Requested experiment: %s in %s is not valid.', - requested_experiment, REQUESTED_EXPERIMENTS_PATH) - return - fuzzers = requested_experiment['fuzzers'] - - benchmark_type = requested_experiment.get('type') - if benchmark_type == benchmark_utils.BenchmarkType.BUG.value: - valid_benchmarks = benchmark_utils.exclude_non_cpp( - benchmark_utils.get_bug_benchmarks()) - else: - valid_benchmarks = benchmark_utils.exclude_non_cpp( - benchmark_utils.get_coverage_benchmarks()) - - benchmarks = requested_experiment.get('benchmarks') - if benchmarks is None: - benchmarks = valid_benchmarks - else: - errors = False - for benchmark in benchmarks: - if benchmark not in valid_benchmarks: - logs.error( - 'Requested experiment:' - ' in %s, %s is not a valid %s benchmark.', - requested_experiment, benchmark, benchmark_type) - errors = True - if errors: - return - - logs.info('Running experiment: %s with fuzzers: %s.', experiment_name, - ' '.join(fuzzers)) - description = _get_description(requested_experiment) - oss_fuzz_corpus = _use_oss_fuzz_corpus(requested_experiment) - _run_experiment(experiment_name, fuzzers, benchmarks, description, - oss_fuzz_corpus, dry_run) - - -def _run_experiment( # pylint: disable=too-many-arguments - experiment_name, - fuzzers, - benchmarks, - description, - oss_fuzz_corpus, - dry_run=False): - """Run an experiment named |experiment_name| on |fuzzer_configs| and shut it - down once it terminates.""" - logs.info('Starting experiment: %s.', experiment_name) - if dry_run: - logs.info('Dry run. Not actually running experiment.') - return - run_experiment.start_experiment(experiment_name, - EXPERIMENT_CONFIG_FILE, - benchmarks, - fuzzers, - description=description, - concurrent_builds=CONCURRENT_BUILDS, - oss_fuzz_corpus=oss_fuzz_corpus) - - -def main(): - """Run an experiment.""" - logs.initialize() - parser = argparse.ArgumentParser(description='Run a requested experiment.') - # TODO(metzman): Add a way to exit immediately if there is already an - # experiment running. FuzzBench's scheduler isn't smart enough to deal with - # this properly. - parser.add_argument('-d', - '--dry-run', - help='Dry run, don\'t actually run the experiment', - default=False, - action='store_true') - args = parser.parse_args() - try: - run_requested_experiment(args.dry_run) - except Exception: # pylint: disable=broad-except - logger.error('Error running requested experiment.') - return 1 - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/service/crontab b/service/crontab deleted file mode 100644 index 5a260689c..000000000 --- a/service/crontab +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Example crontab file that can be used to run the service. Note that the path -# needs to be set because cron jobs will typically use different paths than -# users (it breaks without this). Though the SQL database can only be used by -# Google accounts that have permission, we don't include the password here. - -# m h dom mon dow command -00 1,13 * * * export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/opt/puppetlabs/bin; export POSTGRES_PASSWORD=""; $HOME/fuzzbench/service/run.bash \ No newline at end of file diff --git a/service/test_automatic_run_experiment.py b/service/test_automatic_run_experiment.py deleted file mode 100644 index cd61d42d7..000000000 --- a/service/test_automatic_run_experiment.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for automatic_run_experiment.py""" -import os -import datetime -from unittest import mock - -import pytest - -from common import utils -from service import automatic_run_experiment - -# pylint: disable=invalid-name,unused-argument - -# A valid experiment name. -EXPERIMENT = '2020-01-01' - -EXPERIMENT_REQUESTS = [{ - 'experiment': datetime.date(2020, 6, 8), - 'fuzzers': ['aflplusplus', 'libfuzzer'], -}, { - 'experiment': datetime.date(2020, 6, 5), - 'fuzzers': ['honggfuzz', 'afl'], - 'description': 'Test experiment', - 'oss_fuzz_corpus': True, -}] - - -@mock.patch('experiment.run_experiment.start_experiment') -@mock.patch('common.logs.warning') -@mock.patch('service.automatic_run_experiment._get_requested_experiments') -def test_run_requested_experiment_pause_service( - mocked_get_requested_experiments, mocked_warning, - mocked_start_experiment, db): - """Tests that run_requested_experiment doesn't run an experiment when a - pause is requested.""" - experiment_requests_with_pause = EXPERIMENT_REQUESTS.copy() - experiment_requests_with_pause.append( - automatic_run_experiment.PAUSE_SERVICE_KEYWORD) - mocked_get_requested_experiments.return_value = ( - experiment_requests_with_pause) - - assert (automatic_run_experiment.run_requested_experiment(dry_run=False) is - None) - mocked_warning.assert_called_with( - 'Pause service requested, not running experiment.') - assert mocked_start_experiment.call_count == 0 - - -@mock.patch('experiment.run_experiment.start_experiment') -@mock.patch('service.automatic_run_experiment._get_requested_experiments') -def test_run_requested_experiment(mocked_get_requested_experiments, - mocked_start_experiment, db): - """Tests that run_requested_experiment starts and stops the experiment - properly.""" - mocked_get_requested_experiments.return_value = EXPERIMENT_REQUESTS - expected_experiment_name = '2020-06-05' - expected_fuzzers = ['honggfuzz', 'afl'] - automatic_run_experiment.run_requested_experiment(dry_run=False) - expected_config_file = os.path.join(utils.ROOT_DIR, 'service', - 'experiment-config.yaml') - - expected_benchmarks = sorted([ - 'bloaty_fuzz_target', - 'curl_curl_fuzzer_http', - 'jsoncpp_jsoncpp_fuzzer', - 'libpcap_fuzz_both', - 'libxslt_xpath', - 'mbedtls_fuzz_dtlsclient', - 'openssl_x509', - 'sqlite3_ossfuzz', - 'systemd_fuzz-link-parser', - 'zlib_zlib_uncompress_fuzzer', - 'freetype2_ftfuzzer', - 'harfbuzz_hb-shape-fuzzer', - 'lcms_cms_transform_fuzzer', - 'libjpeg-turbo_libjpeg_turbo_fuzzer', - 'libpng_libpng_read_fuzzer', - 'libxml2_xml', - 'openh264_decoder_fuzzer', - 'openthread_ot-ip6-send-fuzzer', - 'proj4_proj_crs_to_crs_fuzzer', - 're2_fuzzer', - 'stb_stbi_read_fuzzer', - 'vorbis_decode_fuzzer', - 'woff2_convert_woff2ttf_fuzzer', - ]) - expected_call = mock.call( - expected_experiment_name, - expected_config_file, - expected_benchmarks, - expected_fuzzers, - description='Test experiment', - concurrent_builds=(automatic_run_experiment.CONCURRENT_BUILDS), - oss_fuzz_corpus=True) - start_experiment_call_args = mocked_start_experiment.call_args_list - assert len(start_experiment_call_args) == 1 - start_experiment_call_args = start_experiment_call_args[0] - assert start_experiment_call_args == expected_call - - -@pytest.mark.parametrize( - ('name', 'expected_result'), [('02000-1-1', False), ('2020-1-1', False), - ('2020-01-01', True), - ('2020-01-01-aflplusplus', True), - ('2020-01-01-1', True)]) -def test_validate_experiment_name(name, expected_result): - """Tests that validate experiment name returns True for valid names and - False for names that are not valid.""" - assert (automatic_run_experiment.validate_experiment_name(name) == - expected_result) - - -# Call the parameter exp_request instead of request because pytest reserves it. -@pytest.mark.parametrize( - ('exp_request', 'expected_result'), - [ - ({ - 'experiment': EXPERIMENT, - 'fuzzers': ['afl'] - }, True), - # Not a dict. - (1, False), - # No fuzzers. - ({ - 'experiment': EXPERIMENT, - 'fuzzers': [] - }, False), - # No fuzzers. - ({ - 'experiment': EXPERIMENT - }, False), - # No experiment. - ({ - 'fuzzers': ['afl'] - }, False), - # Invalid experiment name for request. - ({ - 'experiment': 'invalid', - 'fuzzers': ['afl'] - }, False), - # Invalid experiment name. - ({ - 'experiment': 'i' * 100, - 'fuzzers': ['afl'] - }, False), - # Nonexistent fuzzers. - ({ - 'experiment': EXPERIMENT, - 'fuzzers': ['nonexistent-fuzzer'] - }, False), - # Invalid fuzzers. - ( - { - 'experiment': EXPERIMENT, - 'fuzzers': ['1'] # Need to make this exist. - }, - False), - # Invalid description. - ({ - 'experiment': EXPERIMENT, - 'fuzzers': ['afl'], - 'description': 1, - }, False), - # Invalid oss_fuzz_corpus flag. - ({ - 'experiment': EXPERIMENT, - 'fuzzers': ['afl'], - 'oss_fuzz_corpus': 'invalid', - }, False), - ]) -def test_validate_experiment_requests(exp_request, expected_result): - """Tests that validate_experiment_requests returns True for valid fuzzres - and False for invalid ones.""" - assert (automatic_run_experiment.validate_experiment_requests([exp_request]) - is expected_result) - - -def test_validate_experiment_requests_duplicate_experiments(): - """Tests that validate_experiment_requests returns False if the experiment - names are duplicated.""" - requests = [ - { - 'experiment': EXPERIMENT, - 'fuzzers': ['afl'] - }, - { - 'experiment': EXPERIMENT, - 'fuzzers': ['libfuzzer'] - }, - ] - assert not automatic_run_experiment.validate_experiment_requests(requests) - - -def test_validate_experiment_requests_one_valid_one_invalid(): - """Tests that validate_experiment_requests returns False even if some - requests are valid.""" - requests = [ - { - 'experiment': EXPERIMENT, - 'fuzzers': ['afl'] - }, - { - 'experiment': '2020-02-02', - 'fuzzers': [] - }, - ] - assert not automatic_run_experiment.validate_experiment_requests(requests)