From 72c16df360022d65d183c332d62f81b188625720 Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Wed, 19 Oct 2022 16:47:54 +1100 Subject: [PATCH] Manually merging #1493 to this branch --- common/gcloud.py | 8 ++++---- common/new_process.py | 6 +++--- docs/reference/benchmarks.py | 10 +++++----- experiment/run_experiment.py | 22 +++++++++++----------- presubmit.py | 3 --- 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/common/gcloud.py b/common/gcloud.py index 3ccc0a5fb..9dd627123 100644 --- a/common/gcloud.py +++ b/common/gcloud.py @@ -16,7 +16,7 @@ import enum import posixpath import subprocess -from typing import List +from typing import List, Optional from common import experiment_utils from common import logs @@ -47,7 +47,7 @@ class InstanceType(enum.Enum): def create_instance(instance_name: str, instance_type: InstanceType, config: dict, - startup_script: str = None, + startup_script: Optional[str] = None, preemptible: bool = False, **kwargs) -> bool: """Creates a GCE instance with name, |instance_name|, type, |instance_type| @@ -126,12 +126,12 @@ def set_default_project(cloud_project: str): ['gcloud', 'config', 'set', 'project', cloud_project]) -def run_local_instance(startup_script: str = None) -> bool: +def run_local_instance(startup_script: Optional[str] = None) -> bool: """Does the equivalent of "create_instance" for local experiments, runs |startup_script| in the background.""" command = ['/bin/bash', startup_script] subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - return new_process.ProcessResult(0, '', False) + return new_process.ProcessResult(0, '', False).retcode == 0 def create_instance_template(template_name, docker_image, env, project, zone): diff --git a/common/new_process.py b/common/new_process.py index dca5df613..957573279 100644 --- a/common/new_process.py +++ b/common/new_process.py @@ -17,7 +17,7 @@ import signal import subprocess import threading -from typing import List +from typing import List, Optional from common import logs @@ -72,10 +72,10 @@ def execute( # pylint: disable=too-many-locals,too-many-branches command: List[str], *args, expect_zero: bool = True, - timeout: int = None, + timeout: Optional[int] = None, write_to_stdout=False, # If not set, will default to PIPE. - output_file=None, + output_file: Optional[int] = None, # Not True by default because we can't always set group on processes. kill_children: bool = False, **kwargs) -> ProcessResult: diff --git a/docs/reference/benchmarks.py b/docs/reference/benchmarks.py index e2e269a01..fdb61006f 100644 --- a/docs/reference/benchmarks.py +++ b/docs/reference/benchmarks.py @@ -25,8 +25,8 @@ from common import benchmark_utils from common import filesystem +from common import fuzzer_config from common import fuzzer_utils as common_fuzzer_utils -from common import oss_fuzz from common import utils from fuzzers import utils as fuzzer_utils @@ -68,10 +68,10 @@ def get_real_benchmark_name(benchmark): if not os.path.isdir(os.path.join(benchmarks_dir, real_benchmark)): continue - if not benchmark_utils.is_oss_fuzz(real_benchmark): + if not benchmark_utils.is_oss_fuzz_benchmark(real_benchmark): continue - config = oss_fuzz.get_config(real_benchmark) + config = fuzzer_config.get_config(real_benchmark) if config['project'] == benchmark: return real_benchmark @@ -126,8 +126,8 @@ def get_binary_size_mb(fuzz_target_path): def get_fuzz_target(benchmark, benchmark_path): """Returns the fuzz target and its path for |benchmark|.""" - if benchmark_utils.is_oss_fuzz(benchmark): - fuzz_target = oss_fuzz.get_config(benchmark)['fuzz_target'] + if benchmark_utils.is_oss_fuzz_benchmark(benchmark): + fuzz_target = fuzzer_config.get_config(benchmark)['fuzz_target'] else: fuzz_target = common_fuzzer_utils.DEFAULT_FUZZ_TARGET_NAME diff --git a/experiment/run_experiment.py b/experiment/run_experiment.py index 413184d6a..e54be01a0 100644 --- a/experiment/run_experiment.py +++ b/experiment/run_experiment.py @@ -22,7 +22,7 @@ import sys import tarfile import tempfile -from typing import Dict, List +from typing import Dict, List, Optional import jinja2 import yaml @@ -232,16 +232,16 @@ def start_experiment( # pylint: disable=too-many-arguments config_filename: str, benchmarks: List[str], fuzzers: List[str], - description: str = None, - no_seeds=False, - no_dictionaries=False, - oss_fuzz_corpus=False, - allow_uncommitted_changes=False, - concurrent_builds=None, - measurers_cpus=None, - runners_cpus=None, - region_coverage=False, - custom_seed_corpus_dir=None): + description: Optional[str] = None, + no_seeds: bool = False, + no_dictionaries: bool = False, + oss_fuzz_corpus: bool = False, + allow_uncommitted_changes: bool = False, + concurrent_builds: Optional[int] = None, + measurers_cpus: Optional[int] = None, + runners_cpus: Optional[int] = None, + region_coverage: bool = False, + custom_seed_corpus_dir: Optional[str] = None): """Start a fuzzer benchmarking experiment.""" if not allow_uncommitted_changes: check_no_uncommitted_changes() diff --git a/presubmit.py b/presubmit.py index 1be78153e..f97f9ed92 100644 --- a/presubmit.py +++ b/presubmit.py @@ -241,9 +241,6 @@ def pytype(paths: List[Path]) -> bool: """Run pytype on |path| if it is a python file. Return False if it fails type checking.""" paths = [path for path in paths if is_python(path)] - if not paths: - return True - base_command = ['python3', '-m', 'pytype'] success = True