From ac6e4d4bf0b1826ebf424ac84c4a9f05ab6e01b0 Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Wed, 12 Oct 2022 15:01:00 +1100
Subject: [PATCH 1/7] Update python to python3.10.7

---
 .github/workflows/fuzzers.yml   | 2 +-
 .github/workflows/presubmit.yml | 2 +-
 Makefile                        | 2 +-
 analysis/benchmark_results.py   | 5 +----
 analysis/experiment_results.py  | 5 +----
 requirements.txt                | 6 +++---
 6 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml
index bfa7f9294..ff2b2f3f7 100644
--- a/.github/workflows/fuzzers.yml
+++ b/.github/workflows/fuzzers.yml
@@ -127,7 +127,7 @@ jobs:
     - name: Setup Python environment
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.10
 
     # Copied from:
     # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions
diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 40c05e91f..7742bdffd 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -21,7 +21,7 @@ jobs:
     - name: Setup Python environment
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.10
 
     # Copied from:
     # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions
diff --git a/Makefile b/Makefile
index 62f2c0ee7..fd11524dd 100644
--- a/Makefile
+++ b/Makefile
@@ -48,7 +48,7 @@ SHELL := /bin/bash
 VENV_ACTIVATE := .venv/bin/activate
 
 ${VENV_ACTIVATE}: requirements.txt
-	python3.9 -m venv .venv || python3 -m venv .venv
+	python3.10 -m venv .venv || python3 -m venv .venv
 	source ${VENV_ACTIVATE} && python3 -m pip install --upgrade pip setuptools && python3 -m pip install -r requirements.txt
 
 install-dependencies: ${VENV_ACTIVATE}
diff --git a/analysis/benchmark_results.py b/analysis/benchmark_results.py
index 178723fdb..a9dd8f918 100644
--- a/analysis/benchmark_results.py
+++ b/analysis/benchmark_results.py
@@ -73,10 +73,7 @@ def _relevant_column(self):
         the analysis (e.g., 'edges_covered', or 'bugs_covered')."""
         return 'edges_covered' if self.type == 'code' else 'bugs_covered'
 
-    @property
-    @functools.lru_cache()
-    # TODO(lszekeres): With python3.8+, replace above two decorators with:
-    # @functools.cached_property
+    @functools.cached_property
     def _benchmark_df(self):
         exp_df = self._experiment_df
         return exp_df[exp_df.benchmark == self.name]
diff --git a/analysis/experiment_results.py b/analysis/experiment_results.py
index b7be0ae6a..444a1587b 100644
--- a/analysis/experiment_results.py
+++ b/analysis/experiment_results.py
@@ -113,10 +113,7 @@ def description_link(commit, fuzzer):
         df.index = df.index.map(lambda fuzzer: description_link(commit, fuzzer))
         return df
 
-    @property
-    @functools.lru_cache()
-    # TODO(lszekeres): With python3.8+, replace above two decorators with:
-    # @functools.cached_property
+    @functools.cached_property
     def _experiment_snapshots_df(self):
         """Data frame containing only the time snapshots, for each benchmark,
         based on which we do further analysis, i.e., statistical tests and
diff --git a/requirements.txt b/requirements.txt
index 544972a33..ddf085f54 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,7 @@ MarkupSafe==2.0.1
 Orange3==3.28.0
 pandas==1.2.4
 psutil==5.9.0
-psycopg2-binary==2.8.4
+psycopg2-binary==2.9.4
 pyfakefs==3.7.1
 pytest==6.1.2
 python-dateutil==2.8.1
@@ -20,12 +20,12 @@ PyYAML==5.4
 redis==3.5.3
 rq==1.4.3
 scikit-posthocs==0.6.2
-scipy==1.6.2
+scipy==1.9.2
 seaborn==0.11.1
 sqlalchemy==1.3.19
 protobuf==3.20.1
 
 # Needed for development.
 pylint==2.7.4
-pytype==2021.4.15
+pytype==2022.9.27
 yapf==0.30.0

From 0b8af63a574043382d7ef6e5c61dfb14cdf768b1 Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Wed, 12 Oct 2022 15:19:16 +1100
Subject: [PATCH 2/7] Update Google-Cloud-SDK installation

---
 docker/base-image/Dockerfile | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile
index 38fe27a6a..b44d171fc 100644
--- a/docker/base-image/Dockerfile
+++ b/docker/base-image/Dockerfile
@@ -45,14 +45,16 @@ RUN cd /tmp/ && \
 COPY ./requirements.txt /
 RUN pip3 install -r /requirements.txt
 
-# Install google-cloud-sdk.
-RUN apt-get update -y && apt-get install -y \
+# Install google-cloud-cli (named google-cloud-sdk prior to version 371.0.0).
+RUN apt-get update && \
+    apt-get install -y \
     apt-transport-https \
-    lsb-release
-RUN CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
-    echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" \
+    ca-certificates \
+    gnupg
+
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
     | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \
-    | apt-key add - && \
-    apt-get update -y && \
-    apt-get install -y google-cloud-sdk
+    | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
+    apt-get update && \
+    apt-get install -y google-cloud-cli

From dfcad778066aafd44b4625b9552ecb247253e33b Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Wed, 12 Oct 2022 15:20:03 +1100
Subject: [PATCH 3/7] Remove previous workaround for ubuntu:xenial

---
 experiment/build/generate_cloudbuild.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/experiment/build/generate_cloudbuild.py b/experiment/build/generate_cloudbuild.py
index 82d31fa68..385110596 100644
--- a/experiment/build/generate_cloudbuild.py
+++ b/experiment/build/generate_cloudbuild.py
@@ -132,16 +132,6 @@ def create_cloudbuild_spec(image_templates,
     """
     cloudbuild_spec = {'steps': [], 'images': []}
 
-    # Workaround for bug https://github.com/moby/moby/issues/40262.
-    # This is only needed for base-image as it inherits from ubuntu:xenial.
-    if build_base_images:
-        cloudbuild_spec['steps'].append({
-            'id': 'pull-ubuntu-xenial',
-            'env': ['DOCKER_BUILDKIT=1'],
-            'name': DOCKER_IMAGE,
-            'args': ['pull', 'ubuntu:xenial'],
-        })
-
     for image_name, image_specs in image_templates.items():
         step = {
             'id': image_name,

From 3d0ff08a519241fb4888a9510b1e93799faadc3e Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Wed, 12 Oct 2022 15:24:08 +1100
Subject: [PATCH 4/7] Update Ubuntu to 22.04 (Jammy Jellyfish)

---
 docker/base-image/Dockerfile                 | 29 ++++----------------
 docker/benchmark-builder/Dockerfile          |  5 ----
 docker/gcb/base-images.yaml                  |  4 +--
 docker/generate_makefile.py                  |  2 +-
 experiment/build/test_generate_cloudbuild.py |  8 +++---
 5 files changed, 12 insertions(+), 36 deletions(-)

diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile
index b44d171fc..684e05a96 100644
--- a/docker/base-image/Dockerfile
+++ b/docker/base-image/Dockerfile
@@ -12,34 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:xenial
+FROM ubuntu:jammy
 
-# Build Python 3.8.6 from source because pandas doesn't support xenial's
-# Python3 version (3.5.2).
-ENV PYTHON_VERSION 3.8.6
-RUN apt-get update -y && apt-get install -y \
-    build-essential \
+# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish)
+ENV PYTHON_VESION 3.10
+RUN apt-get update && apt-get install -y \
     rsync \
     curl \
-    zlib1g-dev \
-    libncurses5-dev \
-    libgdbm-dev \
-    libnss3-dev \
-    libssl-dev \
-    libreadline-dev \
-    libffi-dev \
     virtualenv \
-    libbz2-dev \
-    liblzma-dev \
-    libsqlite3-dev
-
-RUN cd /tmp/ && \
-    curl -O https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz && \
-    tar -xvf Python-$PYTHON_VERSION.tar.xz && \
-    cd Python-$PYTHON_VERSION && \
-    ./configure --enable-loadable-sqlite-extensions --enable-optimizations && \
-    make -j install && \
-    rm -r /tmp/Python-$PYTHON_VERSION.tar.xz /tmp/Python-$PYTHON_VERSION
+    liblzma-dev
 
 # Install common python dependencies.
 COPY ./requirements.txt /
diff --git a/docker/benchmark-builder/Dockerfile b/docker/benchmark-builder/Dockerfile
index 18d7710d8..d289b4c0d 100755
--- a/docker/benchmark-builder/Dockerfile
+++ b/docker/benchmark-builder/Dockerfile
@@ -27,11 +27,6 @@ ENV FUZZER $fuzzer
 ENV BENCHMARK $benchmark
 ENV DEBUG_BUILDER $debug_builder
 
-# Copy latest python3 from base-image into local.
-COPY --from=base-image /usr/local/bin/python3* /usr/local/bin/
-COPY --from=base-image /usr/local/lib/python3.8 /usr/local/lib/python3.8
-COPY --from=base-image /usr/local/include/python3.8 /usr/local/include/python3.8
-COPY --from=base-image /usr/local/lib/python3.8/site-packages /usr/local/lib/python3.8/site-packages
 
 # Copy the entire fuzzers directory tree to allow for module dependencies.
 COPY fuzzers $SRC/fuzzers
diff --git a/docker/gcb/base-images.yaml b/docker/gcb/base-images.yaml
index a4f194bc6..ab40bfb89 100644
--- a/docker/gcb/base-images.yaml
+++ b/docker/gcb/base-images.yaml
@@ -20,10 +20,10 @@ images:
 steps:
 - args:
   - pull
-  - ubuntu:xenial
+  - ubuntu:jammy
   env:
     - DOCKER_BUILDKIT=1
-  id: pull-ubuntu-xenial
+  id: pull-ubuntu-jammy
   name: docker:19.03.12
 - args:
   - build
diff --git a/docker/generate_makefile.py b/docker/generate_makefile.py
index 154c3aeda..63d91611b 100755
--- a/docker/generate_makefile.py
+++ b/docker/generate_makefile.py
@@ -116,7 +116,7 @@ def get_rules_for_image(name, image):
                 section += ' .' + dep
     section += '\n'
     if 'base-' in name:
-        section += '\tdocker pull ubuntu:xenial\n'
+        section += '\tdocker pull ubuntu:jammy\n'
     section += '\tdocker build \\\n'
     section += '\t--tag ' + os.path.join(BASE_TAG, image['tag']) + ' \\\n'
     section += '\t--build-arg BUILDKIT_INLINE_CACHE=1 \\\n'
diff --git a/experiment/build/test_generate_cloudbuild.py b/experiment/build/test_generate_cloudbuild.py
index 07cd03082..bc175a935 100644
--- a/experiment/build/test_generate_cloudbuild.py
+++ b/experiment/build/test_generate_cloudbuild.py
@@ -34,10 +34,10 @@ def test_generate_cloudbuild_spec_build_base_image(experiment):
 
     expected_spec = {
         'steps': [{
-            'id': 'pull-ubuntu-xenial',
+            'id': 'pull-ubuntu-jammy',
             'env': ['DOCKER_BUILDKIT=1'],
             'name': 'docker:19.03.12',
-            'args': ['pull', 'ubuntu:xenial']
+            'args': ['pull', 'ubuntu:jammy']
         }, {
             'id': 'base-image',
             'env': ['DOCKER_BUILDKIT=1'],
@@ -79,10 +79,10 @@ def test_generate_cloudbuild_spec_other_registry(experiment):
 
     expected_spec = {
         'steps': [{
-            'id': 'pull-ubuntu-xenial',
+            'id': 'pull-ubuntu-jammy',
             'env': ['DOCKER_BUILDKIT=1'],
             'name': 'docker:19.03.12',
-            'args': ['pull', 'ubuntu:xenial']
+            'args': ['pull', 'ubuntu:jammy']
         }, {
             'id': 'base-image',
             'env': ['DOCKER_BUILDKIT=1'],

From af5b0f6eddf19df405d7fdc2a083270d7d3bec7e Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Wed, 12 Oct 2022 17:49:46 +1100
Subject: [PATCH 5/7] Fix all errors/warns caused by outdated packages

---
 analysis/data_utils.py                       |  8 +++++---
 analysis/test_data_utils.py                  |  2 +-
 docker/base-image/Dockerfile                 |  2 +-
 experiment/build/test_generate_cloudbuild.py | 10 ----------
 requirements.txt                             | 14 +++++++-------
 5 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/analysis/data_utils.py b/analysis/data_utils.py
index d07bba933..903949316 100644
--- a/analysis/data_utils.py
+++ b/analysis/data_utils.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Utility functions for data (frame) transformations."""
+import pandas as pd
+
 from analysis import stat_tests
 from common import environment
 
@@ -85,7 +87,7 @@ def clobber_experiments_data(df, experiments):
         experiment_pairs = experiment_data[['benchmark',
                                             'fuzzer']].apply(tuple, axis=1)
         to_include = experiment_data[~experiment_pairs.isin(covered_pairs)]
-        result = result.append(to_include)
+        result = pd.concat([result, to_include])
     return result
 
 
@@ -221,7 +223,7 @@ def experiment_summary(experiment_snapshots_df):
 def benchmark_rank_by_mean(benchmark_snapshot_df, key='edges_covered'):
     """Returns ranking of fuzzers based on mean coverage."""
     assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
-    means = benchmark_snapshot_df.groupby('fuzzer')[key].mean()
+    means = benchmark_snapshot_df.groupby('fuzzer')[key].mean().astype(int)
     means.rename('mean cov', inplace=True)
     return means.sort_values(ascending=False)
 
@@ -229,7 +231,7 @@ def benchmark_rank_by_mean(benchmark_snapshot_df, key='edges_covered'):
 def benchmark_rank_by_median(benchmark_snapshot_df, key='edges_covered'):
     """Returns ranking of fuzzers based on median coverage."""
     assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
-    medians = benchmark_snapshot_df.groupby('fuzzer')[key].median()
+    medians = benchmark_snapshot_df.groupby('fuzzer')[key].median().astype(int)
     medians.rename('median cov', inplace=True)
     return medians.sort_values(ascending=False)
 
diff --git a/analysis/test_data_utils.py b/analysis/test_data_utils.py
index 437d1524f..133c8ceba 100644
--- a/analysis/test_data_utils.py
+++ b/analysis/test_data_utils.py
@@ -355,4 +355,4 @@ def test_experiment_rank_by_average_normalized_score():
     pd_test.assert_series_equal(ranking,
                                 expected_ranking,
                                 check_names=False,
-                                check_less_precise=True)
+                                rtol=10**-3)
diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile
index 684e05a96..38154bab2 100644
--- a/docker/base-image/Dockerfile
+++ b/docker/base-image/Dockerfile
@@ -14,7 +14,7 @@
 
 FROM ubuntu:jammy
 
-# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish)
+# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish).
 ENV PYTHON_VESION 3.10
 RUN apt-get update && apt-get install -y \
     rsync \
diff --git a/experiment/build/test_generate_cloudbuild.py b/experiment/build/test_generate_cloudbuild.py
index bc175a935..47152e7a6 100644
--- a/experiment/build/test_generate_cloudbuild.py
+++ b/experiment/build/test_generate_cloudbuild.py
@@ -34,11 +34,6 @@ def test_generate_cloudbuild_spec_build_base_image(experiment):
 
     expected_spec = {
         'steps': [{
-            'id': 'pull-ubuntu-jammy',
-            'env': ['DOCKER_BUILDKIT=1'],
-            'name': 'docker:19.03.12',
-            'args': ['pull', 'ubuntu:jammy']
-        }, {
             'id': 'base-image',
             'env': ['DOCKER_BUILDKIT=1'],
             'name': 'docker:19.03.12',
@@ -79,11 +74,6 @@ def test_generate_cloudbuild_spec_other_registry(experiment):
 
     expected_spec = {
         'steps': [{
-            'id': 'pull-ubuntu-jammy',
-            'env': ['DOCKER_BUILDKIT=1'],
-            'name': 'docker:19.03.12',
-            'args': ['pull', 'ubuntu:jammy']
-        }, {
             'id': 'base-image',
             'env': ['DOCKER_BUILDKIT=1'],
             'name': 'docker:19.03.12',
diff --git a/requirements.txt b/requirements.txt
index ddf085f54..172fceda6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,24 +4,24 @@ google-auth==1.30.1
 google-cloud-error-reporting==1.1.2
 google-cloud-logging==1.15.1
 google-cloud-secret-manager==2.4.0
-clusterfuzz==0.0.1a0
+clusterfuzz==2.5.6
 Jinja2==2.11.3
-numpy==1.22.0
+numpy==1.23.3
 MarkupSafe==2.0.1
 Orange3==3.28.0
-pandas==1.2.4
+pandas==1.5.0
 psutil==5.9.0
 psycopg2-binary==2.9.4
-pyfakefs==3.7.1
-pytest==6.1.2
+pyfakefs==5.0.0
+pytest==7.1.3
 python-dateutil==2.8.1
-pytz==2019.3
+pytz==2022.4
 PyYAML==5.4
 redis==3.5.3
 rq==1.4.3
 scikit-posthocs==0.6.2
 scipy==1.9.2
-seaborn==0.11.1
+seaborn==0.12.0
 sqlalchemy==1.3.19
 protobuf==3.20.1
 

From c90072fa521aa184445584b50f05918225a02c98 Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Wed, 12 Oct 2022 18:43:38 +1100
Subject: [PATCH 6/7] Be more specific about python version

---
 .github/workflows/fuzzers.yml   | 2 +-
 .github/workflows/presubmit.yml | 2 +-
 docker/base-image/Dockerfile    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml
index ff2b2f3f7..c62d729c4 100644
--- a/.github/workflows/fuzzers.yml
+++ b/.github/workflows/fuzzers.yml
@@ -127,7 +127,7 @@ jobs:
     - name: Setup Python environment
       uses: actions/setup-python@v2
       with:
-        python-version: 3.10
+        python-version: 3.10.4
 
     # Copied from:
     # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions
diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 7742bdffd..dd80b6a4d 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -21,7 +21,7 @@ jobs:
     - name: Setup Python environment
       uses: actions/setup-python@v2
       with:
-        python-version: 3.10
+        python-version: 3.10.4
 
     # Copied from:
     # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions
diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile
index 38154bab2..54b207baf 100644
--- a/docker/base-image/Dockerfile
+++ b/docker/base-image/Dockerfile
@@ -14,7 +14,7 @@
 
 FROM ubuntu:jammy
 
-# Python 3.10 is the default version in Ubuntu 22.04 (Jammy Jellyfish).
+# Python 3.10.4 is the default version in Ubuntu 22.04 (Jammy Jellyfish).
 ENV PYTHON_VESION 3.10
 RUN apt-get update && apt-get install -y \
     rsync \

From 0ca7ae0dc9cd7ef1a1e254cff89daa1f228b4854 Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Wed, 12 Oct 2022 18:43:57 +1100
Subject: [PATCH 7/7] Remove redundant env var

---
 docker/base-image/Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile
index 54b207baf..4d9678623 100644
--- a/docker/base-image/Dockerfile
+++ b/docker/base-image/Dockerfile
@@ -15,7 +15,6 @@
 FROM ubuntu:jammy
 
 # Python 3.10.4 is the default version in Ubuntu 22.04 (Jammy Jellyfish).
-ENV PYTHON_VESION 3.10
 RUN apt-get update && apt-get install -y \
     rsync \
     curl \