From ac6e4d4bf0b1826ebf424ac84c4a9f05ab6e01b0 Mon Sep 17 00:00:00 2001 From: Dongge Liu <donggeliu@google.com> Date: Wed, 12 Oct 2022 15:01:00 +1100 Subject: [PATCH 1/7] Update python to python3.10.7 --- .github/workflows/fuzzers.yml | 2 +- .github/workflows/presubmit.yml | 2 +- Makefile | 2 +- analysis/benchmark_results.py | 5 +---- analysis/experiment_results.py | 5 +---- requirements.txt | 6 +++--- 6 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index bfa7f9294..ff2b2f3f7 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -127,7 +127,7 @@ jobs: - name: Setup Python environment uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 # Copied from: # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 40c05e91f..7742bdffd 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Python environment uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 # Copied from: # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions diff --git a/Makefile b/Makefile index 62f2c0ee7..fd11524dd 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ SHELL := /bin/bash VENV_ACTIVATE := .venv/bin/activate ${VENV_ACTIVATE}: requirements.txt - python3.9 -m venv .venv || python3 -m venv .venv + python3.10 -m venv .venv || python3 -m venv .venv source ${VENV_ACTIVATE} && python3 -m pip install --upgrade pip setuptools && python3 -m pip install -r requirements.txt install-dependencies: ${VENV_ACTIVATE} diff --git a/analysis/benchmark_results.py b/analysis/benchmark_results.py index 178723fdb..a9dd8f918 100644 --- a/analysis/benchmark_results.py +++ b/analysis/benchmark_results.py @@ -73,10 +73,7 @@ def _relevant_column(self): the analysis (e.g., 'edges_covered', or 'bugs_covered').""" return 'edges_covered' if self.type == 'code' else 'bugs_covered' - @property - @functools.lru_cache() - # TODO(lszekeres): With python3.8+, replace above two decorators with: - # @functools.cached_property + @functools.cached_property def _benchmark_df(self): exp_df = self._experiment_df return exp_df[exp_df.benchmark == self.name] diff --git a/analysis/experiment_results.py b/analysis/experiment_results.py index b7be0ae6a..444a1587b 100644 --- a/analysis/experiment_results.py +++ b/analysis/experiment_results.py @@ -113,10 +113,7 @@ def description_link(commit, fuzzer): df.index = df.index.map(lambda fuzzer: description_link(commit, fuzzer)) return df - @property - @functools.lru_cache() - # TODO(lszekeres): With python3.8+, replace above two decorators with: - # @functools.cached_property + @functools.cached_property def _experiment_snapshots_df(self): """Data frame containing only the time snapshots, for each benchmark, based on which we do further analysis, i.e., statistical tests and diff --git a/requirements.txt b/requirements.txt index 544972a33..ddf085f54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ MarkupSafe==2.0.1 Orange3==3.28.0 pandas==1.2.4 psutil==5.9.0 -psycopg2-binary==2.8.4 +psycopg2-binary==2.9.4 pyfakefs==3.7.1 pytest==6.1.2 python-dateutil==2.8.1 @@ -20,12 +20,12 @@ PyYAML==5.4 redis==3.5.3 rq==1.4.3 scikit-posthocs==0.6.2 -scipy==1.6.2 +scipy==1.9.2 seaborn==0.11.1 sqlalchemy==1.3.19 protobuf==3.20.1 # Needed for development. pylint==2.7.4 -pytype==2021.4.15 +pytype==2022.9.27 yapf==0.30.0 From 0b8af63a574043382d7ef6e5c61dfb14cdf768b1 Mon Sep 17 00:00:00 2001 From: Dongge Liu <donggeliu@google.com> Date: Wed, 12 Oct 2022 15:19:16 +1100 Subject: [PATCH 2/7] Update Google-Cloud-SDK installation --- docker/base-image/Dockerfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 38fe27a6a..b44d171fc 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -45,14 +45,16 @@ RUN cd /tmp/ && \ COPY ./requirements.txt / RUN pip3 install -r /requirements.txt -# Install google-cloud-sdk. -RUN apt-get update -y && apt-get install -y \ +# Install google-cloud-cli (named google-cloud-sdk prior to version 371.0.0). +RUN apt-get update && \ + apt-get install -y \ apt-transport-https \ - lsb-release -RUN CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" \ + ca-certificates \ + gnupg + +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ - | apt-key add - && \ - apt-get update -y && \ - apt-get install -y google-cloud-sdk + | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \ + apt-get update && \ + apt-get install -y google-cloud-cli From dfcad778066aafd44b4625b9552ecb247253e33b Mon Sep 17 00:00:00 2001 From: Dongge Liu <donggeliu@google.com> Date: Wed, 12 Oct 2022 15:20:03 +1100 Subject: [PATCH 3/7] Remove previous workaround for ubuntu:xenial --- experiment/build/generate_cloudbuild.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/experiment/build/generate_cloudbuild.py b/experiment/build/generate_cloudbuild.py index 82d31fa68..385110596 100644 --- a/experiment/build/generate_cloudbuild.py +++ b/experiment/build/generate_cloudbuild.py @@ -132,16 +132,6 @@ def create_cloudbuild_spec(image_templates, """ cloudbuild_spec = {'steps': [], 'images': []} - # Workaround for bug https://github.com/moby/moby/issues/40262. - # This is only needed for base-image as it inherits from ubuntu:xenial. - if build_base_images: - cloudbuild_spec['steps'].append({ - 'id': 'pull-ubuntu-xenial', - 'env': ['DOCKER_BUILDKIT=1'], - 'name': DOCKER_IMAGE, - 'args': ['pull', 'ubuntu:xenial'], - }) - for image_name, image_specs in image_templates.items(): step = { 'id': image_name, From 3d0ff08a519241fb4888a9510b1e93799faadc3e Mon Sep 17 00:00:00 2001 From: Dongge Liu <donggeliu@google.com> Date: Wed, 12 Oct 2022 15:24:08 +1100 Subject: [PATCH 4/7] Update Ubuntu to 22.04 (Jammy Jellyfish) --- docker/base-image/Dockerfile | 29 ++++---------------- docker/benchmark-builder/Dockerfile | 5 ---- docker/gcb/base-images.yaml | 4 +-- docker/generate_makefile.py | 2 +- experiment/build/test_generate_cloudbuild.py | 8 +++--- 5 files changed, 12 insertions(+), 36 deletions(-) diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index b44d171fc..684e05a96 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -12,34 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:xenial +FROM ubuntu:jammy -# Build Python 3.8.6 from source because pandas doesn't support xenial's -# Python3 version (3.5.2). -ENV PYTHON_VERSION 3.8.6 -RUN apt-get update -y && apt-get install -y \ - build-essential \ +# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish) +ENV PYTHON_VESION 3.10 +RUN apt-get update && apt-get install -y \ rsync \ curl \ - zlib1g-dev \ - libncurses5-dev \ - libgdbm-dev \ - libnss3-dev \ - libssl-dev \ - libreadline-dev \ - libffi-dev \ virtualenv \ - libbz2-dev \ - liblzma-dev \ - libsqlite3-dev - -RUN cd /tmp/ && \ - curl -O https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz && \ - tar -xvf Python-$PYTHON_VERSION.tar.xz && \ - cd Python-$PYTHON_VERSION && \ - ./configure --enable-loadable-sqlite-extensions --enable-optimizations && \ - make -j install && \ - rm -r /tmp/Python-$PYTHON_VERSION.tar.xz /tmp/Python-$PYTHON_VERSION + liblzma-dev # Install common python dependencies. COPY ./requirements.txt / diff --git a/docker/benchmark-builder/Dockerfile b/docker/benchmark-builder/Dockerfile index 18d7710d8..d289b4c0d 100755 --- a/docker/benchmark-builder/Dockerfile +++ b/docker/benchmark-builder/Dockerfile @@ -27,11 +27,6 @@ ENV FUZZER $fuzzer ENV BENCHMARK $benchmark ENV DEBUG_BUILDER $debug_builder -# Copy latest python3 from base-image into local. -COPY --from=base-image /usr/local/bin/python3* /usr/local/bin/ -COPY --from=base-image /usr/local/lib/python3.8 /usr/local/lib/python3.8 -COPY --from=base-image /usr/local/include/python3.8 /usr/local/include/python3.8 -COPY --from=base-image /usr/local/lib/python3.8/site-packages /usr/local/lib/python3.8/site-packages # Copy the entire fuzzers directory tree to allow for module dependencies. COPY fuzzers $SRC/fuzzers diff --git a/docker/gcb/base-images.yaml b/docker/gcb/base-images.yaml index a4f194bc6..ab40bfb89 100644 --- a/docker/gcb/base-images.yaml +++ b/docker/gcb/base-images.yaml @@ -20,10 +20,10 @@ images: steps: - args: - pull - - ubuntu:xenial + - ubuntu:jammy env: - DOCKER_BUILDKIT=1 - id: pull-ubuntu-xenial + id: pull-ubuntu-jammy name: docker:19.03.12 - args: - build diff --git a/docker/generate_makefile.py b/docker/generate_makefile.py index 154c3aeda..63d91611b 100755 --- a/docker/generate_makefile.py +++ b/docker/generate_makefile.py @@ -116,7 +116,7 @@ def get_rules_for_image(name, image): section += ' .' + dep section += '\n' if 'base-' in name: - section += '\tdocker pull ubuntu:xenial\n' + section += '\tdocker pull ubuntu:jammy\n' section += '\tdocker build \\\n' section += '\t--tag ' + os.path.join(BASE_TAG, image['tag']) + ' \\\n' section += '\t--build-arg BUILDKIT_INLINE_CACHE=1 \\\n' diff --git a/experiment/build/test_generate_cloudbuild.py b/experiment/build/test_generate_cloudbuild.py index 07cd03082..bc175a935 100644 --- a/experiment/build/test_generate_cloudbuild.py +++ b/experiment/build/test_generate_cloudbuild.py @@ -34,10 +34,10 @@ def test_generate_cloudbuild_spec_build_base_image(experiment): expected_spec = { 'steps': [{ - 'id': 'pull-ubuntu-xenial', + 'id': 'pull-ubuntu-jammy', 'env': ['DOCKER_BUILDKIT=1'], 'name': 'docker:19.03.12', - 'args': ['pull', 'ubuntu:xenial'] + 'args': ['pull', 'ubuntu:jammy'] }, { 'id': 'base-image', 'env': ['DOCKER_BUILDKIT=1'], @@ -79,10 +79,10 @@ def test_generate_cloudbuild_spec_other_registry(experiment): expected_spec = { 'steps': [{ - 'id': 'pull-ubuntu-xenial', + 'id': 'pull-ubuntu-jammy', 'env': ['DOCKER_BUILDKIT=1'], 'name': 'docker:19.03.12', - 'args': ['pull', 'ubuntu:xenial'] + 'args': ['pull', 'ubuntu:jammy'] }, { 'id': 'base-image', 'env': ['DOCKER_BUILDKIT=1'], From af5b0f6eddf19df405d7fdc2a083270d7d3bec7e Mon Sep 17 00:00:00 2001 From: Dongge Liu <donggeliu@google.com> Date: Wed, 12 Oct 2022 17:49:46 +1100 Subject: [PATCH 5/7] Fix all errors/warns caused by outdated packages --- analysis/data_utils.py | 8 +++++--- analysis/test_data_utils.py | 2 +- docker/base-image/Dockerfile | 2 +- experiment/build/test_generate_cloudbuild.py | 10 ---------- requirements.txt | 14 +++++++------- 5 files changed, 14 insertions(+), 22 deletions(-) diff --git a/analysis/data_utils.py b/analysis/data_utils.py index d07bba933..903949316 100644 --- a/analysis/data_utils.py +++ b/analysis/data_utils.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Utility functions for data (frame) transformations.""" +import pandas as pd + from analysis import stat_tests from common import environment @@ -85,7 +87,7 @@ def clobber_experiments_data(df, experiments): experiment_pairs = experiment_data[['benchmark', 'fuzzer']].apply(tuple, axis=1) to_include = experiment_data[~experiment_pairs.isin(covered_pairs)] - result = result.append(to_include) + result = pd.concat([result, to_include]) return result @@ -221,7 +223,7 @@ def experiment_summary(experiment_snapshots_df): def benchmark_rank_by_mean(benchmark_snapshot_df, key='edges_covered'): """Returns ranking of fuzzers based on mean coverage.""" assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!' - means = benchmark_snapshot_df.groupby('fuzzer')[key].mean() + means = benchmark_snapshot_df.groupby('fuzzer')[key].mean().astype(int) means.rename('mean cov', inplace=True) return means.sort_values(ascending=False) @@ -229,7 +231,7 @@ def benchmark_rank_by_mean(benchmark_snapshot_df, key='edges_covered'): def benchmark_rank_by_median(benchmark_snapshot_df, key='edges_covered'): """Returns ranking of fuzzers based on median coverage.""" assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!' - medians = benchmark_snapshot_df.groupby('fuzzer')[key].median() + medians = benchmark_snapshot_df.groupby('fuzzer')[key].median().astype(int) medians.rename('median cov', inplace=True) return medians.sort_values(ascending=False) diff --git a/analysis/test_data_utils.py b/analysis/test_data_utils.py index 437d1524f..133c8ceba 100644 --- a/analysis/test_data_utils.py +++ b/analysis/test_data_utils.py @@ -355,4 +355,4 @@ def test_experiment_rank_by_average_normalized_score(): pd_test.assert_series_equal(ranking, expected_ranking, check_names=False, - check_less_precise=True) + rtol=10**-3) diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 684e05a96..38154bab2 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -14,7 +14,7 @@ FROM ubuntu:jammy -# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish) +# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish). ENV PYTHON_VESION 3.10 RUN apt-get update && apt-get install -y \ rsync \ diff --git a/experiment/build/test_generate_cloudbuild.py b/experiment/build/test_generate_cloudbuild.py index bc175a935..47152e7a6 100644 --- a/experiment/build/test_generate_cloudbuild.py +++ b/experiment/build/test_generate_cloudbuild.py @@ -34,11 +34,6 @@ def test_generate_cloudbuild_spec_build_base_image(experiment): expected_spec = { 'steps': [{ - 'id': 'pull-ubuntu-jammy', - 'env': ['DOCKER_BUILDKIT=1'], - 'name': 'docker:19.03.12', - 'args': ['pull', 'ubuntu:jammy'] - }, { 'id': 'base-image', 'env': ['DOCKER_BUILDKIT=1'], 'name': 'docker:19.03.12', @@ -79,11 +74,6 @@ def test_generate_cloudbuild_spec_other_registry(experiment): expected_spec = { 'steps': [{ - 'id': 'pull-ubuntu-jammy', - 'env': ['DOCKER_BUILDKIT=1'], - 'name': 'docker:19.03.12', - 'args': ['pull', 'ubuntu:jammy'] - }, { 'id': 'base-image', 'env': ['DOCKER_BUILDKIT=1'], 'name': 'docker:19.03.12', diff --git a/requirements.txt b/requirements.txt index ddf085f54..172fceda6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,24 +4,24 @@ google-auth==1.30.1 google-cloud-error-reporting==1.1.2 google-cloud-logging==1.15.1 google-cloud-secret-manager==2.4.0 -clusterfuzz==0.0.1a0 +clusterfuzz==2.5.6 Jinja2==2.11.3 -numpy==1.22.0 +numpy==1.23.3 MarkupSafe==2.0.1 Orange3==3.28.0 -pandas==1.2.4 +pandas==1.5.0 psutil==5.9.0 psycopg2-binary==2.9.4 -pyfakefs==3.7.1 -pytest==6.1.2 +pyfakefs==5.0.0 +pytest==7.1.3 python-dateutil==2.8.1 -pytz==2019.3 +pytz==2022.4 PyYAML==5.4 redis==3.5.3 rq==1.4.3 scikit-posthocs==0.6.2 scipy==1.9.2 -seaborn==0.11.1 +seaborn==0.12.0 sqlalchemy==1.3.19 protobuf==3.20.1 From c90072fa521aa184445584b50f05918225a02c98 Mon Sep 17 00:00:00 2001 From: Dongge Liu <donggeliu@google.com> Date: Wed, 12 Oct 2022 18:43:38 +1100 Subject: [PATCH 6/7] Be more specific about python version --- .github/workflows/fuzzers.yml | 2 +- .github/workflows/presubmit.yml | 2 +- docker/base-image/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index ff2b2f3f7..c62d729c4 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -127,7 +127,7 @@ jobs: - name: Setup Python environment uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: 3.10.4 # Copied from: # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 7742bdffd..dd80b6a4d 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Python environment uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: 3.10.4 # Copied from: # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 38154bab2..54b207baf 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -14,7 +14,7 @@ FROM ubuntu:jammy -# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish). +# Python 3.10.4 is the default version in Ubuntu 22.04 (Jammy Jellyfish). ENV PYTHON_VESION 3.10 RUN apt-get update && apt-get install -y \ rsync \ From 0ca7ae0dc9cd7ef1a1e254cff89daa1f228b4854 Mon Sep 17 00:00:00 2001 From: Dongge Liu <donggeliu@google.com> Date: Wed, 12 Oct 2022 18:43:57 +1100 Subject: [PATCH 7/7] Remove redundant env var --- docker/base-image/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 54b207baf..4d9678623 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -15,7 +15,6 @@ FROM ubuntu:jammy # Python 3.10.4 is the default version in Ubuntu 22.04 (Jammy Jellyfish). -ENV PYTHON_VESION 3.10 RUN apt-get update && apt-get install -y \ rsync \ curl \