From 9fdf46df3bc55c237bbdb5e223da61b99aa2518c Mon Sep 17 00:00:00 2001 From: Ivan Yurchenko Date: Wed, 30 Nov 2022 07:57:17 +0200 Subject: [PATCH] dumper: ensure dumping can work without having GDB in the container The main theme of this commit is to ensure the dumper works fine without GDB installed in the target namespace/container. Besides that, the commit: 1. Introduces integration testing with three more Linux distributions--Fedora, Debian, Ubuntu--in addition to Alpine Linux. 2. Does some refactoring on the dumper side. Closes #200 Closes #209 --- Makefile | 28 +-- README.md | 27 +- integration_tests/Makefile | 50 ++-- integration_tests/e2e_docker/.gitignore | 18 +- integration_tests/e2e_docker/Dockerfile | 9 - .../e2e_docker/alpine.Dockerfile | 24 ++ .../e2e_docker/debian.Dockerfile | 24 ++ .../e2e_docker/fedora.Dockerfile | 29 +++ .../e2e_docker/ubuntu.Dockerfile | 32 +++ integration_tests/manual_test_e2e.py | 22 +- pyheap/src/docker.py | 48 ++++ pyheap/src/gdb.py | 97 ++++++++ pyheap/src/mount.py | 65 +++++ pyheap/src/namespaces.py | 126 ++++++++++ pyheap/src/pyheap_dump.py | 231 ++++++++---------- .../tests/unit/test_gdb_solid_search_paths.py | 213 ++++++++++++++++ .../tests/{ => unit}/test_type_assumptions.py | 0 test_inferiors/Dockerfile | 17 -- test_inferiors/Makefile | 26 ++ test_inferiors/README.md | 7 +- test_inferiors/alpine.Dockerfile | 44 ++++ test_inferiors/debian.Dockerfile | 30 +++ 22 files changed, 954 insertions(+), 213 deletions(-) delete mode 100644 integration_tests/e2e_docker/Dockerfile create mode 100644 integration_tests/e2e_docker/alpine.Dockerfile create mode 100644 integration_tests/e2e_docker/debian.Dockerfile create mode 100644 integration_tests/e2e_docker/fedora.Dockerfile create mode 100644 integration_tests/e2e_docker/ubuntu.Dockerfile create mode 100644 pyheap/src/docker.py create mode 100644 pyheap/src/gdb.py create mode 100644 pyheap/src/mount.py create mode 100644 pyheap/src/namespaces.py create mode 100644 pyheap/tests/unit/test_gdb_solid_search_paths.py rename pyheap/tests/{ => unit}/test_type_assumptions.py (100%) delete mode 100644 test_inferiors/Dockerfile create mode 100644 test_inferiors/Makefile create mode 100644 test_inferiors/alpine.Dockerfile create mode 100644 test_inferiors/debian.Dockerfile diff --git a/Makefile b/Makefile index 60c42ad..11f0a71 100644 --- a/Makefile +++ b/Makefile @@ -18,43 +18,43 @@ clean: (cd pyheap && $(MAKE) clean) -.PHONY: integration_tests -integration_tests: integration_tests_3_8 integration_tests_3_9 integration_tests_3_10 integration_tests_3_11 +.PHONY: integration-tests +integration-tests: integration-tests-3-8 integration-tests-3-9 integration-tests-3-10 integration-tests-3-11 pyheap/dist/pyheap_dump.pyz: (cd pyheap && $(MAKE) dist) -.PHONY: integration_tests_3_8 -integration_tests_3_8: pyheap/dist/pyheap_dump.pyz +.PHONY: integration-tests-3-8 +integration-tests-3-8: pyheap/dist/pyheap_dump.pyz (cd integration_tests && \ - $(MAKE) test_target_docker_image_3_8 && \ + $(MAKE) test-target-docker-images-3-8 && \ PYENV_VERSION=3.8 poetry env use python && \ poetry run pip install -e ../pyheap-ui/ && \ poetry install && \ poetry run pytest -vv ./*.py) -.PHONY: integration_tests_3_9 -integration_tests_3_9: pyheap/dist/pyheap_dump.pyz +.PHONY: integration-tests-3-9 +integration-tests-3-9: pyheap/dist/pyheap_dump.pyz (cd integration_tests && \ - $(MAKE) test_target_docker_image_3_9 && \ + $(MAKE) test-target-docker-images-3-9 && \ PYENV_VERSION=3.9 poetry env use python && \ poetry run pip install -e ../pyheap-ui/ && \ poetry install && \ poetry run pytest -vv ./*.py) -.PHONY: integration_tests_3_10 -integration_tests_3_10: pyheap/dist/pyheap_dump.pyz +.PHONY: integration-tests-3-10 +integration-tests-3-10: pyheap/dist/pyheap_dump.pyz (cd integration_tests && \ - $(MAKE) test_target_docker_image_3_10 && \ + $(MAKE) test-target-docker-images-3-10 && \ PYENV_VERSION=3.10 poetry env use python && \ poetry run pip install -e ../pyheap-ui/ && \ poetry install && \ poetry run pytest -vv ./*.py) -.PHONY: integration_tests_3_11 -integration_tests_3_11: pyheap/dist/pyheap_dump.pyz +.PHONY: integration-tests-3-11 +integration-tests-3-11: pyheap/dist/pyheap_dump.pyz (cd integration_tests && \ - $(MAKE) test_target_docker_image_3_11 && \ + $(MAKE) test-target-docker-images-3-11 && \ PYENV_VERSION=3.11 poetry env use python && \ poetry run pip install -e ../pyheap-ui/ && \ poetry install && \ diff --git a/README.md b/README.md index 88f9408..9ccad35 100644 --- a/README.md +++ b/README.md @@ -3,16 +3,27 @@ A heap dumper and analyzer for CPython based on GDB. The product consists of two parts: -1. The zero-dependency dumper script. +1. The dumper which uses GDB. 2. The Flask-based UI for heap dump visualization. +## Requirements + +The dumper needs the following: +1. GDB must be installed where the dumper runs (e.g. on the machine host), but is not needed near a target process (e.g. in a container). +2. CPython 3.8 - 3.11. +3. Docker CLI for working with Docker containers directly (e.g. calling `docker inspect`). + ## Compatibility -The dumper is compatible with a target process running on: -- CPython 3.8; -- CPython 3.9; -- CPython 3.10; -- CPython 3.11. +**Only Linux** is supported at the moment. + +The dumper is compatible with a target process running on CPython 3.8 - 3.11. + +The target process were tested in the following OSes: +- Alpine Linux; +- Ubuntu; +- Fedora; +- Debian. Some popular libraries were tested: - Django; @@ -55,8 +66,6 @@ $ sudo python3 pyheap_dump.pyz --docker-container --file heap.p If it's not the root process in the container, or you work with another container system (e.g. systemd-nspawn) or just generic Linux namespaces, you need to find the target PID. Please mind that this must be the PID from the dumper point of view: processes in namespaces can have their own PID numbers. For example, if you're about to run the dumper on a Linux host and the target process is running in a container, check the process list with `ps` or `top` on the host. Use `--pid/-p` for the dumper. -Make sure the GDB executable is available in the target mount namespace. If the target is in a Docker container, you most likely need to install GDB inside it (please note this can be done in a running container as well). - If the target process is running under a different user (normal for Docker), you need to use `sudo` with `python3 pyheap_dump.pyz ...`. PyHeap dumper will automatically transfer the heap file from the target namespace to the specified location. @@ -169,7 +178,7 @@ Currently, the dumper sees objects traced by the CPython garbage collector and t Integration tests run on CI. However, end-to-end tests that use the real GDB cannot be run in GitHub Actions. You can run them locally using ```bash -make clean integration_tests +make clean integration-tests ``` You need [pyenv](https://github.com/pyenv/pyenv) with Python 3.8, 3.9, 3.10, and 3.11 installed and [Poetry](https://python-poetry.org/). diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 9d4b64c..30e05a0 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -18,29 +18,41 @@ clean: rm e2e_docker/inferior-simple.py -.PHONY: test_target_docker_image_3_8 -test_target_docker_image_3_8: e2e_docker/inferior-simple.py +define build_image docker build e2e_docker \ - --build-arg BASE_VERSION="3.8.15-bullseye" \ - --tag ivanyu/pyheap-e2e-test-target:3.8 + -f "e2e_docker/$2.Dockerfile" \ + --build-arg BASE_IMAGE_VERSION="$3" \ + --build-arg PYHEAP_PYTHON_VERSION="$1" \ + --tag "ivanyu/pyheap-e2e-test-target:$2-$1" +endef -.PHONY: test_target_docker_image_3_9 -test_target_docker_image_3_9: e2e_docker/inferior-simple.py - docker build e2e_docker \ - --build-arg BASE_VERSION="3.9.15-bullseye" \ - --tag ivanyu/pyheap-e2e-test-target:3.9 +.PHONY: test-target-docker-images-3-8 +test-target-docker-images-3-8: e2e_docker/inferior-simple.py + $(call build_image,3.8,alpine,3.8.15-alpine3.16) + $(call build_image,3.8,debian,3.8.15-slim-bullseye) + $(call build_image,3.8,ubuntu,22.04) + $(call build_image,3.8,fedora,36) -.PHONY: test_target_docker_image_3_10 -test_target_docker_image_3_10: e2e_docker/inferior-simple.py - docker build e2e_docker \ - --build-arg BASE_VERSION="3.10.8-bullseye" \ - --tag ivanyu/pyheap-e2e-test-target:3.10 +.PHONY: test-target-docker-images-3-9 +test-target-docker-images-3-9: e2e_docker/inferior-simple.py + $(call build_image,3.9,alpine,3.9.15-alpine3.16) + $(call build_image,3.9,debian,3.9.15-slim-bullseye) + $(call build_image,3.9,ubuntu,22.04) + $(call build_image,3.9,fedora,36) -.PHONY: test_target_docker_image_3_11 -test_target_docker_image_3_11: e2e_docker/inferior-simple.py - docker build e2e_docker \ - --build-arg BASE_VERSION="3.11.0-bullseye" \ - --tag ivanyu/pyheap-e2e-test-target:3.11 +.PHONY: test-target-docker-images-3-10 +test-target-docker-images-3-10: e2e_docker/inferior-simple.py + $(call build_image,3.10,alpine,3.10.8-alpine3.16) + $(call build_image,3.10,debian,3.10.8-slim-bullseye) + $(call build_image,3.10,ubuntu,22.04) + $(call build_image,3.10,fedora,36) + +.PHONY: test-target-docker-images-3-11 +test-target-docker-images-3-11: e2e_docker/inferior-simple.py + $(call build_image,3.11,alpine,3.11.0-alpine3.16) + $(call build_image,3.11,debian,3.11.0-slim-bullseye) + $(call build_image,3.11,ubuntu,22.04) + $(call build_image,3.11,fedora,36) e2e_docker/inferior-simple.py: cp ../test_inferiors/inferior-simple.py $@ diff --git a/integration_tests/e2e_docker/.gitignore b/integration_tests/e2e_docker/.gitignore index 5d0f124..9cb2a1b 100644 --- a/integration_tests/e2e_docker/.gitignore +++ b/integration_tests/e2e_docker/.gitignore @@ -1,2 +1,16 @@ -* -!Dockerfile +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +inferior-simple.py diff --git a/integration_tests/e2e_docker/Dockerfile b/integration_tests/e2e_docker/Dockerfile deleted file mode 100644 index f5d38b0..0000000 --- a/integration_tests/e2e_docker/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG BASE_VERSION -FROM python:$BASE_VERSION - -RUN apt-get update && apt-get install -y gdb \ - && rm -rf /var/lib/apt/lists/* - -COPY inferior-simple.py /inferior-simple.py - -CMD ["python", "/inferior-simple.py"] diff --git a/integration_tests/e2e_docker/alpine.Dockerfile b/integration_tests/e2e_docker/alpine.Dockerfile new file mode 100644 index 0000000..9fe34a1 --- /dev/null +++ b/integration_tests/e2e_docker/alpine.Dockerfile @@ -0,0 +1,24 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +ARG BASE_IMAGE_VERSION +FROM python:$BASE_IMAGE_VERSION + +ARG PYHEAP_PYTHON_VERSION + +COPY inferior-simple.py /inferior-simple.py + +RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python +CMD ["/test-python", "/inferior-simple.py"] diff --git a/integration_tests/e2e_docker/debian.Dockerfile b/integration_tests/e2e_docker/debian.Dockerfile new file mode 100644 index 0000000..9fe34a1 --- /dev/null +++ b/integration_tests/e2e_docker/debian.Dockerfile @@ -0,0 +1,24 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +ARG BASE_IMAGE_VERSION +FROM python:$BASE_IMAGE_VERSION + +ARG PYHEAP_PYTHON_VERSION + +COPY inferior-simple.py /inferior-simple.py + +RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python +CMD ["/test-python", "/inferior-simple.py"] diff --git a/integration_tests/e2e_docker/fedora.Dockerfile b/integration_tests/e2e_docker/fedora.Dockerfile new file mode 100644 index 0000000..f44cea6 --- /dev/null +++ b/integration_tests/e2e_docker/fedora.Dockerfile @@ -0,0 +1,29 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +ARG BASE_IMAGE_VERSION +FROM fedora:$BASE_IMAGE_VERSION + +ARG PYHEAP_PYTHON_VERSION + +RUN dnf -y update && dnf install -y \ + "python${PYHEAP_PYTHON_VERSION}" \ + which \ + && dnf clean all + +COPY inferior-simple.py /inferior-simple.py + +RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python +CMD ["/test-python", "/inferior-simple.py"] diff --git a/integration_tests/e2e_docker/ubuntu.Dockerfile b/integration_tests/e2e_docker/ubuntu.Dockerfile new file mode 100644 index 0000000..e27d2da --- /dev/null +++ b/integration_tests/e2e_docker/ubuntu.Dockerfile @@ -0,0 +1,32 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +ARG BASE_IMAGE_VERSION +FROM ubuntu:$BASE_IMAGE_VERSION + +ARG PYHEAP_PYTHON_VERSION + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC +RUN apt-get update \ + && apt-get install -y software-properties-common \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get install -y "python${PYHEAP_PYTHON_VERSION}" \ + && rm -rf /var/lib/apt/lists/* + +COPY inferior-simple.py /inferior-simple.py + +RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python +CMD ["/test-python", "/inferior-simple.py"] diff --git a/integration_tests/manual_test_e2e.py b/integration_tests/manual_test_e2e.py index c4f93b4..143f37f 100644 --- a/integration_tests/manual_test_e2e.py +++ b/integration_tests/manual_test_e2e.py @@ -13,23 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import json import mmap import os import subprocess import sys import time from contextlib import contextmanager, closing -from typing import Iterator, Union +from typing import Iterator, Union, Optional import pytest from _pytest.tmpdir import TempPathFactory from pyheap_ui.heap_reader import HeapReader -@pytest.mark.parametrize("docker", [False, True]) -def test_e2e(docker: bool, test_heap_path: str) -> None: - with _inferior_process(docker) as ip_pid_or_container, _dumper_process( - test_heap_path, ip_pid_or_container, docker +@pytest.mark.parametrize("docker_base", ["alpine", "debian", "ubuntu", "fedora", None]) +def test_e2e(docker_base: Optional[str], test_heap_path: str) -> None: + is_docker = docker_base is not None + with _inferior_process(docker_base) as ip_pid_or_container, _dumper_process( + test_heap_path, ip_pid_or_container, is_docker ) as dp: print(f"Inferior process/container {ip_pid_or_container}") print(f"Dumper process {dp.pid}") @@ -72,7 +72,7 @@ def _inferior_process_plain() -> Iterator[int]: @contextmanager -def _inferior_process_docker() -> Iterator[str]: +def _inferior_process_docker(docker_base: str) -> Iterator[str]: python_version = f"{sys.version_info.major}.{sys.version_info.minor}" docker_proc = subprocess.run( [ @@ -80,7 +80,7 @@ def _inferior_process_docker() -> Iterator[str]: "run", "--rm", "--detach", - f"ivanyu/pyheap-e2e-test-target:{python_version}", + f"ivanyu/pyheap-e2e-test-target:{docker_base}-{python_version}", ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -103,9 +103,9 @@ def _inferior_process_docker() -> Iterator[str]: @contextmanager -def _inferior_process(docker: bool) -> Iterator[Union[int, str]]: - if docker: - with _inferior_process_docker() as r: +def _inferior_process(docker_base: Optional[str]) -> Iterator[Union[int, str]]: + if docker_base is not None: + with _inferior_process_docker(docker_base) as r: yield r else: with _inferior_process_plain() as r: diff --git a/pyheap/src/docker.py b/pyheap/src/docker.py new file mode 100644 index 0000000..b40ec5b --- /dev/null +++ b/pyheap/src/docker.py @@ -0,0 +1,48 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +import json +import subprocess + + +def get_container_pid(container: str) -> int: + proc = subprocess.run( + ["docker", "inspect", container], + text=True, + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if proc.returncode != 0: + raise Exception( + f"Cannot determine target PID: " + f"`docker inspect {container}` returned: {proc.stderr}" + ) + + inspect_obj = json.loads(proc.stdout) + if len(inspect_obj) != 1: + raise Exception( + f"Cannot determine target PID: " + f"Expected 1 object in `docker inspect {container}`, but got {len(inspect_obj)}" + ) + + state = inspect_obj[0]["State"] + if state["Status"] != "running": + raise Exception(f"Cannot determine target PID: Container is not running") + + pid = int(state["Pid"]) + return pid diff --git a/pyheap/src/gdb.py b/pyheap/src/gdb.py new file mode 100644 index 0000000..d8961f9 --- /dev/null +++ b/pyheap/src/gdb.py @@ -0,0 +1,97 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +import contextlib +import os +import re +from pathlib import Path +from typing import List, Optional, Iterator + +import mount + + +def solib_search_paths(pid: int, pid_in_ns: int) -> List[str]: + # libc (isn't used in e.g. Alpine Linux) and libpthread (maybe not loaded) are optional. + libc_path: Optional[str] = None + libpthread_path: Optional[str] = None + try: + with open(f"/proc/{pid}/maps", "r") as f: + for l in f.readlines(): + parts = re.split("\s+", l.strip()) + if len(parts) != 6: + continue + + path = parts[-1] + if libc_path is None and re.search(r"libc(-[\d.]+)?\.so(\.|$)", path): + libc_path = path + if libpthread_path is None and re.search( + r"libpthread(-[\d.]+)?\.so(\.|$)", path + ): + libpthread_path = path + except PermissionError as e: + print(e) + print("Hint: the target process is likely run under a different user, use sudo") + raise e + + dirs = set() + if libc_path is not None: + dirs.add(str(Path(libc_path).parent)) + if libpthread_path is not None: + dirs.add(str(Path(libpthread_path).parent)) + return [f"/proc/{pid_in_ns}/root{d}" for d in dirs] + + +@contextlib.contextmanager +def bind_gdb_exe(gdb_exe: str, temp_dir: str) -> Iterator[str]: + gdb_mount_file = Path(temp_dir) / "gdb" + gdb_mount_file.touch( + mode=os.stat(gdb_exe).st_mode & 0o777, + exist_ok=False, + ) + mounted = str(gdb_mount_file) + mount.mount(gdb_exe, mounted, "", mountflags=mount.MS_BIND) + try: + yield mounted + finally: + mount.umount(mounted) + + +def shadow_target_exe_dir_for_gdb(target_pid: int, temp_dir: str) -> None: + """Shadows the target executable directory for GDB. + + There may be a situation, where ``self/pid/exe`` points to an executable inside + the target mount namespace, but which also exists in the dumper/GDB namespace. For example, ``/usr/bin/python3.11``. + GDB (``canonicalize_file_name`` inside it) doesn't handle this nicely and uses the file in the dumper namespace + for reading symbols. This doesn't work well. If GDB/``canonicalize_file_name`` sees the file doesn't exist, + it reads from the ``self/pid/exe`` directly, which works fine. + + This function basically bind-mounts an empty directory over the target file parent directory (e.g. ``/usr/bin``) in + the dumper namespace. + """ + target_exe = os.readlink(f"/proc/{target_pid}/exe") + if os.path.exists(target_exe): + print( + f"Target exe link resolves to {target_exe}, which exists in our namespace" + ) + else: + return + dir_to_shadow = str(Path(target_exe).parent) + print(f"Will shadow directory {dir_to_shadow}") + + shadow_dir = os.path.join(temp_dir, "shadow") + os.mkdir(shadow_dir, mode=os.stat(dir_to_shadow).st_mode & 0o777) + mount.mount(shadow_dir, dir_to_shadow, "", mountflags=mount.MS_BIND) diff --git a/pyheap/src/mount.py b/pyheap/src/mount.py new file mode 100644 index 0000000..e24f30b --- /dev/null +++ b/pyheap/src/mount.py @@ -0,0 +1,65 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import ctypes +import os +from ctypes.util import find_library +from typing import AnyStr + +# linux/mount.h +MS_PRIVATE = 1 << 18 +MS_NOSUID = 2 # Ignore suid and sgid bits +MS_NODEV = 4 # Disallow access to device special files +MS_NOEXEC = 8 # Disallow program execution +MS_BIND = 4096 +MS_REC = 16384 + + +libc = ctypes.CDLL(find_library("c"), use_errno=True) +libc.mount.argtypes = [ + ctypes.c_char_p, # source + ctypes.c_char_p, # target + ctypes.c_char_p, # filesystem_type + ctypes.c_ulong, # mount_flags + ctypes.c_void_p, # data +] +libc.umount.argtypes = [ctypes.c_char_p] # target + + +def set_propagation_on_root(mountflags: int) -> None: + if libc.mount(b"none", b"/", None, mountflags, None) != 0: + raise Exception( + f"Failed on mount when setting propagation: {os.strerror(ctypes.get_errno())}" + ) + + +def mount(source: AnyStr, target: AnyStr, fs_type: AnyStr, mountflags: int) -> None: + if isinstance(source, str): + source = source.encode("utf-8") + if isinstance(target, str): + target = target.encode("utf-8") + if isinstance(fs_type, str): + fs_type = fs_type.encode("utf-8") + if libc.mount(source, target, fs_type, mountflags, None) != 0: + raise Exception( + f"Failed on mount {source} -> {target} with FS type {fs_type}: {os.strerror(ctypes.get_errno())}" + ) + + +def umount(target: AnyStr) -> None: + if isinstance(target, str): + target = target.encode("utf-8") + if libc.umount(target) != 0: + raise Exception(f"Failed on umount {target}: {os.strerror(ctypes.get_errno())}") diff --git a/pyheap/src/namespaces.py b/pyheap/src/namespaces.py new file mode 100644 index 0000000..1124e50 --- /dev/null +++ b/pyheap/src/namespaces.py @@ -0,0 +1,126 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +import contextlib +import os +import ctypes +from ctypes.util import find_library +from typing import Iterator, Union + +from mount import ( + set_propagation_on_root, + MS_PRIVATE, + MS_REC, + mount, + MS_NODEV, + MS_NOEXEC, + MS_NOSUID, +) + +libc = ctypes.CDLL(find_library("c"), use_errno=True) + +# linux/sched.h +CLONE_NEWNS = 0x00020000 +CLONE_NEWPID = 0x20000000 + + +def nsenter_to_pid_ns_with_fork(pid: int) -> None: + """Does similar to ``nsenter -t -p``. + + In other words, it enters the PID namespace of the specified process and forks. + The child continues. The parent waits for the child and propagates its return code. + """ + + # nsenter into the PID namespace + with _pid_namespace(pid) as fd: + if libc.setns(fd, CLONE_NEWPID) != 0: + raise Exception(f"Failed on setns: {os.strerror(ctypes.get_errno())}") + + # Fork for the PID namespace to take effect. (See `man setns`). + fork_pid = os.fork() + if fork_pid < 0: + # This won't be probably executed as fork() will raise OSError, + # but still having it for completeness. + raise Exception("Fork failed") + elif fork_pid > 0: + # --- The parent waits on the child here --- + + _, child_status = os.waitpid(fork_pid, 0) + kill_signal = child_status & 0xFF + if kill_signal != 0: + print(f"Child killed by signal {kill_signal}") + exit(1) + else: + return_code = child_status & 0xFF00 + exit(return_code) + + # --- The child continues here --- + + +def unshare_and_mount_proc() -> None: + """Does similar to ``unshare --mount-proc``. + + In other words, this creates a new mount namespace and mounts /proc there. + """ + + # Unshare with new mount namespace. + if libc.unshare(CLONE_NEWNS) != 0: + raise Exception(f"Failed on unshare: {os.strerror(ctypes.get_errno())}") + + # Make mount propagation private. + set_propagation_on_root(mountflags=MS_REC | MS_PRIVATE) + + # Mount /proc. + mount( + source="proc", + target="/proc", + fs_type="proc", + mountflags=MS_NOSUID | MS_NOEXEC | MS_NODEV, + ) + + +@contextlib.contextmanager +def _pid_namespace(pid: int) -> Iterator[int]: + fd = os.open(f"/proc/{pid}/ns/pid", os.O_RDONLY) + yield fd + os.close(fd) + + +def two_processes_in_same_pid_namespace( + pid1: Union[int, str], pid2: Union[int, str] +) -> bool: + return _read_pid_namespace_link(pid1) == _read_pid_namespace_link(pid2) + + +def _read_pid_namespace_link(pid: Union[int, str]) -> str: + try: + return os.readlink(f"/proc/{pid}/ns/pid") + except PermissionError as e: + print(e) + print("Hint: the target process is likely run under a different user, use sudo") + raise e + + +def pid_in_own_namespace(pid: Union[int, str]) -> int: + """Finds the PID of a process in its own PID namespace.""" + with open(f"/proc/{pid}/status", "r") as f: + for line in f.readlines(): + line = line.strip() + if line.startswith("NStgid"): + return int(line.split("\t")[-1].strip()) + else: + raise Exception("Cannot determine target process PID in its namespace") diff --git a/pyheap/src/pyheap_dump.py b/pyheap/src/pyheap_dump.py index f7e441d..f75437b 100644 --- a/pyheap/src/pyheap_dump.py +++ b/pyheap/src/pyheap_dump.py @@ -19,59 +19,102 @@ import json import os.path import shutil -import subprocess import uuid -from contextlib import closing +from contextlib import closing, ExitStack from dataclasses import dataclass from importlib.machinery import SourceFileLoader from subprocess import Popen -import tempfile +from tempfile import TemporaryDirectory import time from pathlib import Path -from typing import Optional, Dict, Any, Callable, Union, Tuple - - -def dump_heap(args: argparse.Namespace) -> None: +from typing import ( + Optional, + Dict, + Any, + Callable, + Union, + Tuple, + cast, + ContextManager, +) + +from docker import get_container_pid +from gdb import solib_search_paths, bind_gdb_exe, shadow_target_exe_dir_for_gdb +from namespaces import ( + unshare_and_mount_proc, + nsenter_to_pid_ns_with_fork, + two_processes_in_same_pid_namespace, + pid_in_own_namespace, +) + + +def dump_heap(args: argparse.Namespace) -> int: target_pid: int - gdb_pid: int + target_pid_in_ns: int nsenter_needed: bool if args.docker_container is not None: print("Target is Docker container") - target_pid = _get_container_pid(args.docker_container) - gdb_pid = 1 + target_pid = get_container_pid(args.docker_container) + target_pid_in_ns = 1 + print( + f"Target process PID: {target_pid}, in its own namespace: {target_pid_in_ns}" + ) nsenter_needed = True - elif _pid_namespace(args.pid) == _pid_namespace(os.getpid()): + elif two_processes_in_same_pid_namespace(args.pid, os.getpid()): print("Dumper and target are in same PID namespace") target_pid = args.pid - gdb_pid = target_pid + target_pid_in_ns = target_pid nsenter_needed = False else: target_pid = args.pid - gdb_pid = _target_pid_in_own_namespace(args.pid) - print(f"Target process PID in its namespace: {gdb_pid}") + target_pid_in_ns = pid_in_own_namespace(target_pid) + print(f"Target process PID in its own namespace: {target_pid_in_ns}") nsenter_needed = True - print(f"Dumping heap from process {target_pid} into {args.file}") - print(f"Max length of string representation is {args.str_repr_len}") + solid_search_paths = ":".join(solib_search_paths(target_pid, target_pid_in_ns)) injector_code = _load_code("injector.py") dumper_code = _prepare_dumper_code() - tmp_dir: CrossNamespaceTmpDir - progress_file: CrossNamespaceFile - heap_file: CrossNamespaceFile - with closing(CrossNamespaceTmpDir(target_pid)) as tmp_dir, closing( - tmp_dir.create_file("progress.json", 0o600) - ) as progress_file, closing( - tmp_dir.create_file(f"{uuid.uuid4()}.pyheap", 0o600) - ) as heap_file: - cmd = [] + if nsenter_needed: + nsenter_to_pid_ns_with_fork(target_pid) + unshare_and_mount_proc() + + with ExitStack() as stack: + dumper_temp_dir = stack.enter_context(TemporaryDirectory(prefix="pyheap-")) + + gdb_exe = os.path.realpath(shutil.which("gdb")) if nsenter_needed: - cmd += ["nsenter", "-t", str(target_pid), "-a"] - cmd += [ - "gdb", + gdb_exe = stack.enter_context( + cast(ContextManager[str], bind_gdb_exe(gdb_exe, dumper_temp_dir)) + ) + + if nsenter_needed: + shadow_target_exe_dir_for_gdb(target_pid_in_ns, dumper_temp_dir) + + target_temp_dir = stack.enter_context( + closing(TargetTemporaryDirectory(target_pid_in_ns)) + ) + progress_file = target_temp_dir.create_file("progress.json", 0o600) + heap_file = target_temp_dir.create_file(f"{uuid.uuid4()}.pyheap", 0o600) + + # TODO exlpore solib-absolute-prefix vs solib-search-path + + print(f"Dumping heap from process {target_pid} into {args.file}") + print(f"Max length of string representation is {args.str_repr_len}") + + cmd = [ + gdb_exe, "--readnow", "-iex", + "set verbose on", + "-iex", + f"set sysroot /proc/{target_pid_in_ns}/root", + "-iex", + f"set auto-load safe-path {solid_search_paths}", + "-iex", + f"set solib-search-path {solid_search_paths}", + "-iex", "set debuginfod enabled off", "-ex", "break _PyEval_EvalFrameDefault", @@ -86,13 +129,13 @@ def dump_heap(args: argparse.Namespace) -> None: "-ex", "set max-value-size unlimited", "-ex", - f'set $dump_success = $dump_python_heap("{dumper_code}", "{heap_file.target_path}", {args.str_repr_len}, "{progress_file.target_path}")', + f'set $dump_success = $dump_python_heap("{dumper_code}", "{heap_file}", {args.str_repr_len}, "{progress_file}")', "-ex", "detach", "-ex", "quit $dump_success", "-p", - str(gdb_pid), + str(target_pid_in_ns), ] p = Popen(cmd, shell=False) progress_tracker = ProgressTracker( @@ -101,65 +144,12 @@ def dump_heap(args: argparse.Namespace) -> None: progress_tracker.track_progress() p.communicate() - if p.returncode != 0: - print("Dumping finished with error") - else: + if p.returncode == 0: _move_heap_file(heap_file, args.file) - - exit(p.returncode) - - -def _get_container_pid(container: str) -> int: - proc = subprocess.run( - ["docker", "inspect", container], - text=True, - encoding="utf-8", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if proc.returncode != 0: - print("Cannot determine target PID:") - print(f"`docker inspect {container}` returned:") - print(proc.stderr) - exit(1) - - inspect_obj = json.loads(proc.stdout) - if len(inspect_obj) != 1: - print("Cannot determine target PID:") - print( - f"Expected 1 object in `docker inspect {container}`, but got {len(inspect_obj)}" - ) - exit(1) - - state = inspect_obj[0]["State"] - if state["Status"] != "running": - print("Cannot determine target PID:") - print(f"Container is not running") - exit(1) - - pid = int(state["Pid"]) - print(f"Target PID: {pid}") - return pid - - -def _pid_namespace(pid: Union[int, str]) -> str: - try: - return os.readlink(f"/proc/{pid}/ns/pid") - except PermissionError as e: - print(e) - print("Hint: the target process is likely run under a different user, use sudo") - exit(1) - - -def _target_pid_in_own_namespace(target_pid: Union[int, str]) -> int: - with open(f"/proc/{target_pid}/status", "r") as f: - for l in f.readlines(): - l = l.strip() - if l.startswith("NStgid"): - return int(l.split("\t")[-1].strip()) + return 0 else: - print("Cannot determine target process PID in its namespace") - exit(1) + print("Dumping finished with error") + return p.returncode def _load_code(filename: str) -> str: @@ -191,14 +181,14 @@ def from_json(progress_json: Dict[str, Any]) -> Progress: ) -class CrossNamespaceTmpDir: +class TargetTemporaryDirectory: def __init__(self, target_pid: int) -> None: self._target_root_fs = f"/proc/{target_pid}/root" self._target_uid, self._target_gid = self._target_fs_uid_gid(target_pid) - self._path = tempfile.mkdtemp( + self._tempdir = TemporaryDirectory( prefix="pyheap-", dir=f"{self._target_root_fs}/tmp" ) - os.chown(self._path, self._target_uid, self._target_gid) + os.chown(self._tempdir.name, self._target_uid, self._target_gid) @staticmethod def _target_fs_uid_gid(target_pid: Union[int, str]) -> Tuple[int, int]: @@ -217,46 +207,19 @@ def _target_fs_uid_gid(target_pid: Union[int, str]) -> Tuple[int, int]: else: raise Exception("Cannot determine target process FS UID and GID") - def create_file(self, name: str, mode: int) -> CrossNamespaceFile: - dumper_path = os.path.join(self._path, name) - path_obj = Path(dumper_path) - path_obj.touch(mode=mode, exist_ok=False) - os.chown(dumper_path, self._target_uid, self._target_gid) - target_path = "/" + str(path_obj.relative_to(self._target_root_fs)) - return CrossNamespaceFile(dumper_path=dumper_path, target_path=target_path) + def create_file(self, name: str, mode: int) -> str: + path = Path(self._tempdir.name) / name + path.touch(mode=mode, exist_ok=False) + os.chown(str(path), self._target_uid, self._target_gid) + return str(path) def close(self) -> None: - try: - os.rmdir(self._path) - except OSError as e: # not exist or not empty - print(f"Cannot delete {self._path}: {e}") - - -class CrossNamespaceFile: - def __init__(self, dumper_path: str, target_path: str) -> None: - self._dumper_path = dumper_path - self._target_path = target_path - - @property - def dumper_path(self) -> str: - return self._dumper_path - - @property - def target_path(self) -> str: - return self._target_path - - def close(self) -> None: - try: - os.remove(self._dumper_path) - except FileNotFoundError: - pass # it's ok if it doesn't exist already - except OSError as e: - print(f"Cannot delete {self._dumper_path}: {e}") + self._tempdir.cleanup() class ProgressTracker: def __init__( - self, *, progress_file: CrossNamespaceFile, should_continue: Callable[[], bool] + self, *, progress_file: str, should_continue: Callable[[], bool] ) -> None: self._progress_file = progress_file self._should_continue = should_continue @@ -279,7 +242,7 @@ def track_progress(self) -> None: def _read_progress(self) -> Optional[Progress]: progress: Optional[Progress] = None - progress_file_path = self._progress_file.dumper_path + progress_file_path = self._progress_file try: with open(progress_file_path, "r") as f: content = f.read() @@ -304,14 +267,14 @@ def _display(self, progress: Progress) -> None: ) -def _move_heap_file(heap_file: CrossNamespaceFile, final_path: str) -> None: +def _move_heap_file(heap_file: str, final_path: str) -> None: final_path_unambiguous = final_path i = -1 while os.path.exists(final_path_unambiguous): i += 1 final_path_unambiguous = f"{final_path}.{i}" - print(f"Moving from {heap_file.dumper_path} to {final_path_unambiguous}") - shutil.move(heap_file.dumper_path, final_path_unambiguous) + print(f"Moving from {heap_file} to {final_path_unambiguous}") + shutil.move(heap_file, final_path_unambiguous) print(f"Heap file saved: {final_path_unambiguous}") @@ -335,7 +298,17 @@ def main() -> None: parser.set_defaults(func=dump_heap) args = parser.parse_args() - args.func(args) + try: + return_code = args.func(args) + except SystemExit as e: + exit(e.code) + except: + import traceback + + traceback.print_exc() + exit(1) + else: + exit(return_code) if __name__ == "__main__": diff --git a/pyheap/tests/unit/test_gdb_solid_search_paths.py b/pyheap/tests/unit/test_gdb_solid_search_paths.py new file mode 100644 index 0000000..b6003b7 --- /dev/null +++ b/pyheap/tests/unit/test_gdb_solid_search_paths.py @@ -0,0 +1,213 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional +from unittest.mock import mock_open, patch, MagicMock + +import pytest + +from gdb import solib_search_paths + + +def test_both_path_present_and_same_dir() -> None: + with patch( + "builtins.open", + _mock_maps( + libc_path="/lib/x86_64-linux-gnu/libc-2.31.so", + libpthread_path="/lib/x86_64-linux-gnu/libpthread-2.31.so", + ), + ): + r = solib_search_paths(123, 456) + assert r == ["/proc/456/root/lib/x86_64-linux-gnu"] + + +def test_both_path_present_and_different_dir() -> None: + with patch( + "builtins.open", + _mock_maps( + libc_path="/lib/x86_64-linux-gnu/libc-2.31.so", + libpthread_path="/lib64/libpthread-2.31.so", + ), + ): + r = solib_search_paths(123, 456) + assert set(r) == { + "/proc/456/root/lib/x86_64-linux-gnu", + "/proc/456/root/lib64", + } + + +def test_only_libc_path() -> None: + with patch( + "builtins.open", + _mock_maps( + libc_path="/lib/x86_64-linux-gnu/libc-2.31.so", + libpthread_path=None, + ), + ): + r = solib_search_paths(123, 456) + assert r == ["/proc/456/root/lib/x86_64-linux-gnu"] + + +@pytest.mark.parametrize( + "libc_path", + [ + "/lib/x86_64-linux-gnu/libc-2.31.so", + "/lib/x86_64-linux-gnu/libc.so", + "/lib/x86_64-linux-gnu/libc.so.6", + ], +) +def test_filename_format(libc_path: str) -> None: + with patch( + "builtins.open", + _mock_maps( + libc_path=libc_path, + libpthread_path=None, + ), + ): + r = solib_search_paths(123, 456) + assert r == ["/proc/456/root/lib/x86_64-linux-gnu"] + + +def test_libc_path_not_present() -> None: + with patch( + "builtins.open", + _mock_maps(libc_path=None, libpthread_path=None), + ): + solib_search_paths(123, 456) + + +def _mock_maps( + *, libc_path: Optional[str], libpthread_path: Optional[str] +) -> MagicMock: + maps_data = """558e62a04000-558e62a05000 r--p 00000000 00:1d 28059 /usr/local/bin/python3.10 +558e62a05000-558e62a06000 r-xp 00001000 00:1d 28059 /usr/local/bin/python3.10 +558e62a06000-558e62a07000 r--p 00002000 00:1d 28059 /usr/local/bin/python3.10 +558e62a07000-558e62a08000 r--p 00002000 00:1d 28059 /usr/local/bin/python3.10 +558e62a08000-558e62a09000 rw-p 00003000 00:1d 28059 /usr/local/bin/python3.10 +558e647d4000-558e64b37000 rw-p 00000000 00:00 0 [heap] +7f7008000000-7f70085f5000 rw-p 00000000 00:00 0 +7f70085f5000-7f700c000000 ---p 00000000 00:00 0 +7f700cd37000-7f700ce37000 rw-p 00000000 00:00 0 +7f700d179000-7f700d279000 rw-p 00000000 00:00 0 +7f700d30a000-7f700d40a000 rw-p 00000000 00:00 0 +7f700d482000-7f700d484000 r--p 00000000 00:1d 1384 /usr/lib/x86_64-linux-gnu/libuuid.so.1.3.0 +7f700d484000-7f700d488000 r-xp 00002000 00:1d 1384 /usr/lib/x86_64-linux-gnu/libuuid.so.1.3.0 +7f700d488000-7f700d489000 r--p 00006000 00:1d 1384 /usr/lib/x86_64-linux-gnu/libuuid.so.1.3.0 +7f700d489000-7f700d48a000 r--p 00006000 00:1d 1384 /usr/lib/x86_64-linux-gnu/libuuid.so.1.3.0 +7f700d48a000-7f700d48b000 rw-p 00007000 00:1d 1384 /usr/lib/x86_64-linux-gnu/libuuid.so.1.3.0 +7f700d493000-7f700d495000 r--p 00000000 00:1d 28826 /usr/local/lib/python3.10/lib-dynload/select.cpython-310-x86_64-linux-gnu.so +7f700d495000-7f700d498000 r-xp 00002000 00:1d 28826 /usr/local/lib/python3.10/lib-dynload/select.cpython-310-x86_64-linux-gnu.so +7f700d498000-7f700d49a000 r--p 00005000 00:1d 28826 /usr/local/lib/python3.10/lib-dynload/select.cpython-310-x86_64-linux-gnu.so +7f700d49a000-7f700d49b000 r--p 00006000 00:1d 28826 /usr/local/lib/python3.10/lib-dynload/select.cpython-310-x86_64-linux-gnu.so +7f700d49b000-7f700d49c000 rw-p 00007000 00:1d 28826 /usr/local/lib/python3.10/lib-dynload/select.cpython-310-x86_64-linux-gnu.so +7f700d49c000-7f700d49e000 r--p 00000000 00:1d 28791 /usr/local/lib/python3.10/lib-dynload/_posixsubprocess.cpython-310-x86_64-linux-gnu.so +7f700d49e000-7f700d4a1000 r-xp 00002000 00:1d 28791 /usr/local/lib/python3.10/lib-dynload/_posixsubprocess.cpython-310-x86_64-linux-gnu.so +7f700d4a1000-7f700d4a2000 r--p 00005000 00:1d 28791 /usr/local/lib/python3.10/lib-dynload/_posixsubprocess.cpython-310-x86_64-linux-gnu.so +7f700d4a2000-7f700d4a3000 r--p 00005000 00:1d 28791 /usr/local/lib/python3.10/lib-dynload/_posixsubprocess.cpython-310-x86_64-linux-gnu.so +7f700d4a3000-7f700d4a4000 rw-p 00006000 00:1d 28791 /usr/local/lib/python3.10/lib-dynload/_posixsubprocess.cpython-310-x86_64-linux-gnu.so +7f700d4a4000-7f700d4a5000 r--p 00000000 00:1d 28817 /usr/local/lib/python3.10/lib-dynload/fcntl.cpython-310-x86_64-linux-gnu.so +7f700d4a5000-7f700d4a7000 r-xp 00001000 00:1d 28817 /usr/local/lib/python3.10/lib-dynload/fcntl.cpython-310-x86_64-linux-gnu.so +7f700d4a7000-7f700d4a9000 r--p 00003000 00:1d 28817 /usr/local/lib/python3.10/lib-dynload/fcntl.cpython-310-x86_64-linux-gnu.so +7f700d4a9000-7f700d4aa000 r--p 00004000 00:1d 28817 /usr/local/lib/python3.10/lib-dynload/fcntl.cpython-310-x86_64-linux-gnu.so +7f700d4aa000-7f700d4ab000 rw-p 00005000 00:1d 28817 /usr/local/lib/python3.10/lib-dynload/fcntl.cpython-310-x86_64-linux-gnu.so +7f700d4ab000-7f700d4b0000 r--p 00000000 00:1d 28775 /usr/local/lib/python3.10/lib-dynload/_datetime.cpython-310-x86_64-linux-gnu.so +7f700d4b0000-7f700d4c2000 r-xp 00005000 00:1d 28775 /usr/local/lib/python3.10/lib-dynload/_datetime.cpython-310-x86_64-linux-gnu.so +7f700d4c2000-7f700d4c8000 r--p 00017000 00:1d 28775 /usr/local/lib/python3.10/lib-dynload/_datetime.cpython-310-x86_64-linux-gnu.so +7f700d4c8000-7f700d4c9000 r--p 0001c000 00:1d 28775 /usr/local/lib/python3.10/lib-dynload/_datetime.cpython-310-x86_64-linux-gnu.so +7f700d4c9000-7f700d4cc000 rw-p 0001d000 00:1d 28775 /usr/local/lib/python3.10/lib-dynload/_datetime.cpython-310-x86_64-linux-gnu.so +7f700d4cc000-7f700d5cc000 rw-p 00000000 00:00 0 +7f700d5cc000-7f700d5cd000 ---p 00000000 00:00 0 +7f700d5cd000-7f700e0c2000 rw-p 00000000 00:00 0 +7f700e0c8000-7f700e0cb000 r--p 00000000 00:1d 28802 /usr/local/lib/python3.10/lib-dynload/_struct.cpython-310-x86_64-linux-gnu.so +7f700e0cb000-7f700e0d0000 r-xp 00003000 00:1d 28802 /usr/local/lib/python3.10/lib-dynload/_struct.cpython-310-x86_64-linux-gnu.so +7f700e0d0000-7f700e0d4000 r--p 00008000 00:1d 28802 /usr/local/lib/python3.10/lib-dynload/_struct.cpython-310-x86_64-linux-gnu.so +7f700e0d4000-7f700e0d5000 r--p 0000b000 00:1d 28802 /usr/local/lib/python3.10/lib-dynload/_struct.cpython-310-x86_64-linux-gnu.so +7f700e0d5000-7f700e0d6000 rw-p 0000c000 00:1d 28802 /usr/local/lib/python3.10/lib-dynload/_struct.cpython-310-x86_64-linux-gnu.so +7f700e0d6000-7f700e0d9000 r--p 00000000 00:1d 28819 /usr/local/lib/python3.10/lib-dynload/math.cpython-310-x86_64-linux-gnu.so +7f700e0d9000-7f700e0e1000 r-xp 00003000 00:1d 28819 /usr/local/lib/python3.10/lib-dynload/math.cpython-310-x86_64-linux-gnu.so +7f700e0e1000-7f700e0e5000 r--p 0000b000 00:1d 28819 /usr/local/lib/python3.10/lib-dynload/math.cpython-310-x86_64-linux-gnu.so +7f700e0e5000-7f700e0e6000 r--p 0000e000 00:1d 28819 /usr/local/lib/python3.10/lib-dynload/math.cpython-310-x86_64-linux-gnu.so +7f700e0e6000-7f700e0e7000 rw-p 0000f000 00:1d 28819 /usr/local/lib/python3.10/lib-dynload/math.cpython-310-x86_64-linux-gnu.so +7f700e0e7000-7f700e2e7000 rw-p 00000000 00:00 0 +7f700e2e7000-7f700e2f4000 r--p 00000000 00:1d 617 /lib/x86_64-linux-gnu/libm-2.31.so +7f700e2f4000-7f700e38e000 r-xp 0000d000 00:1d 617 /lib/x86_64-linux-gnu/libm-2.31.so +7f700e38e000-7f700e429000 r--p 000a7000 00:1d 617 /lib/x86_64-linux-gnu/libm-2.31.so +7f700e429000-7f700e42a000 r--p 00141000 00:1d 617 /lib/x86_64-linux-gnu/libm-2.31.so +7f700e42a000-7f700e42b000 rw-p 00142000 00:1d 617 /lib/x86_64-linux-gnu/libm-2.31.so +""" + if libc_path is not None: + maps_data += f"""7f700e42b000-7f700e44d000 r--p 00000000 00:1d 596 {libc_path} +7f700e44d000-7f700e5a7000 r-xp 00022000 00:1d 596 {libc_path} +7f700e5a7000-7f700e5f6000 r--p 0017c000 00:1d 596 {libc_path} +7f700e5f6000-7f700e5fa000 r--p 001ca000 00:1d 596 {libc_path} +7f700e5fa000-7f700e5fc000 rw-p 001ce000 00:1d 596 {libc_path} +7f700e5fc000-7f700e600000 rw-p 00000000 00:00 0 +""" + + maps_data += f"""7f700e600000-7f700e659000 r--p 00000000 00:1d 28227 /usr/local/lib/libpython3.10.so.1.0 +7f700e659000-7f700e87e000 r-xp 00059000 00:1d 28227 /usr/local/lib/libpython3.10.so.1.0 +7f700e87e000-7f700e976000 r--p 0027e000 00:1d 28227 /usr/local/lib/libpython3.10.so.1.0 +7f700e976000-7f700e97b000 r--p 00375000 00:1d 28227 /usr/local/lib/libpython3.10.so.1.0 +7f700e97b000-7f700e9ae000 rw-p 0037a000 00:1d 28227 /usr/local/lib/libpython3.10.so.1.0 +7f700e9ae000-7f700e9b4000 rw-p 00000000 00:00 0 +7f700e9b5000-7f700e9b6000 r--p 00000000 00:1d 28809 /usr/local/lib/python3.10/lib-dynload/_uuid.cpython-310-x86_64-linux-gnu.so +7f700e9b6000-7f700e9b7000 r-xp 00001000 00:1d 28809 /usr/local/lib/python3.10/lib-dynload/_uuid.cpython-310-x86_64-linux-gnu.so +7f700e9b7000-7f700e9b8000 r--p 00002000 00:1d 28809 /usr/local/lib/python3.10/lib-dynload/_uuid.cpython-310-x86_64-linux-gnu.so +7f700e9b8000-7f700e9b9000 r--p 00002000 00:1d 28809 /usr/local/lib/python3.10/lib-dynload/_uuid.cpython-310-x86_64-linux-gnu.so +7f700e9b9000-7f700e9ba000 rw-p 00003000 00:1d 28809 /usr/local/lib/python3.10/lib-dynload/_uuid.cpython-310-x86_64-linux-gnu.so +7f700e9ba000-7f700e9bb000 r--p 00000000 00:1d 28788 /usr/local/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so +7f700e9bb000-7f700e9bc000 r-xp 00001000 00:1d 28788 /usr/local/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so +7f700e9bc000-7f700e9bd000 r--p 00002000 00:1d 28788 /usr/local/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so +7f700e9bd000-7f700e9be000 r--p 00002000 00:1d 28788 /usr/local/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so +7f700e9be000-7f700e9bf000 rw-p 00003000 00:1d 28788 /usr/local/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so +7f700e9bf000-7f700e9c1000 r--p 00000000 00:1d 28782 /usr/local/lib/python3.10/lib-dynload/_json.cpython-310-x86_64-linux-gnu.so +7f700e9c1000-7f700e9c7000 r-xp 00002000 00:1d 28782 /usr/local/lib/python3.10/lib-dynload/_json.cpython-310-x86_64-linux-gnu.so +7f700e9c7000-7f700e9c9000 r--p 00008000 00:1d 28782 /usr/local/lib/python3.10/lib-dynload/_json.cpython-310-x86_64-linux-gnu.so +7f700e9c9000-7f700e9ca000 r--p 00009000 00:1d 28782 /usr/local/lib/python3.10/lib-dynload/_json.cpython-310-x86_64-linux-gnu.so +7f700e9ca000-7f700e9cb000 rw-p 0000a000 00:1d 28782 /usr/local/lib/python3.10/lib-dynload/_json.cpython-310-x86_64-linux-gnu.so +7f700e9cb000-7f700ea20000 r--p 00000000 00:1d 1025 /usr/lib/locale/C.UTF-8/LC_CTYPE +7f700ea20000-7f700ea25000 rw-p 00000000 00:00 0 +7f700ea25000-7f700ea26000 r--p 00000000 00:1d 657 /lib/x86_64-linux-gnu/libutil-2.31.so +7f700ea26000-7f700ea27000 r-xp 00001000 00:1d 657 /lib/x86_64-linux-gnu/libutil-2.31.so +7f700ea27000-7f700ea28000 r--p 00002000 00:1d 657 /lib/x86_64-linux-gnu/libutil-2.31.so +7f700ea28000-7f700ea29000 r--p 00002000 00:1d 657 /lib/x86_64-linux-gnu/libutil-2.31.so +7f700ea29000-7f700ea2a000 rw-p 00003000 00:1d 657 /lib/x86_64-linux-gnu/libutil-2.31.so +7f700ea2a000-7f700ea2b000 r--p 00000000 00:1d 604 /lib/x86_64-linux-gnu/libdl-2.31.so +7f700ea2b000-7f700ea2d000 r-xp 00001000 00:1d 604 /lib/x86_64-linux-gnu/libdl-2.31.so +7f700ea2d000-7f700ea2e000 r--p 00003000 00:1d 604 /lib/x86_64-linux-gnu/libdl-2.31.so +7f700ea2e000-7f700ea2f000 r--p 00003000 00:1d 604 /lib/x86_64-linux-gnu/libdl-2.31.so +7f700ea2f000-7f700ea30000 rw-p 00004000 00:1d 604 /lib/x86_64-linux-gnu/libdl-2.31.so +""" + if libpthread_path is not None: + maps_data += f"""7f700ea30000-7f700ea36000 r--p 00000000 00:1d 641 {libpthread_path} +7f700ea36000-7f700ea46000 r-xp 00006000 00:1d 641 {libpthread_path} +7f700ea46000-7f700ea4c000 r--p 00016000 00:1d 641 {libpthread_path} +7f700ea4c000-7f700ea4d000 r--p 0001b000 00:1d 641 {libpthread_path} +7f700ea4d000-7f700ea4e000 rw-p 0001c000 00:1d 641 {libpthread_path} +""" + maps_data += """7f700ea4e000-7f700ea54000 rw-p 00000000 00:00 0 +7f700ea55000-7f700ea5c000 r--s 00000000 00:1d 1305 /usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache +7f700ea5c000-7f700ea5d000 r--p 00000000 00:1d 584 /lib/x86_64-linux-gnu/ld-2.31.so +7f700ea5d000-7f700ea7d000 r-xp 00001000 00:1d 584 /lib/x86_64-linux-gnu/ld-2.31.so +7f700ea7d000-7f700ea85000 r--p 00021000 00:1d 584 /lib/x86_64-linux-gnu/ld-2.31.so +7f700ea86000-7f700ea87000 r--p 00029000 00:1d 584 /lib/x86_64-linux-gnu/ld-2.31.so +7f700ea87000-7f700ea88000 rw-p 0002a000 00:1d 584 /lib/x86_64-linux-gnu/ld-2.31.so +7f700ea88000-7f700ea89000 rw-p 00000000 00:00 0 +7ffe71ec0000-7ffe71ee1000 rw-p 00000000 00:00 0 [stack] +7ffe71f6d000-7ffe71f71000 r--p 00000000 00:00 0 [vvar] +7ffe71f71000-7ffe71f73000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall]""" + + return mock_open(read_data=maps_data) diff --git a/pyheap/tests/test_type_assumptions.py b/pyheap/tests/unit/test_type_assumptions.py similarity index 100% rename from pyheap/tests/test_type_assumptions.py rename to pyheap/tests/unit/test_type_assumptions.py diff --git a/test_inferiors/Dockerfile b/test_inferiors/Dockerfile deleted file mode 100644 index 8180069..0000000 --- a/test_inferiors/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.10.8-bullseye - -RUN groupadd -g 10042 testgroup && useradd --uid 10123 testuser -g testgroup --create-home - -RUN apt-get update && apt-get install -y gdb \ - && rm -rf /var/lib/apt/lists/* - -USER testuser - -RUN curl -sSL https://install.python-poetry.org | python3 - - -COPY ./ /inferiors/ -WORKDIR /inferiors - -ENV PATH="/home/testuser/.local/bin:$PATH" - -RUN poetry install diff --git a/test_inferiors/Makefile b/test_inferiors/Makefile new file mode 100644 index 0000000..b566d16 --- /dev/null +++ b/test_inferiors/Makefile @@ -0,0 +1,26 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: docker-images +docker-images: docker-image-alpine docker-image-debian + +.PHONY: docker-image-alpine +docker-image-alpine: + docker build . -f alpine.Dockerfile -t ivanyu/pyheap-test-inferiors:alpine + +.PHONY: docker-image-debian +docker-image-debian: + docker build . -f debian.Dockerfile -t ivanyu/pyheap-test-inferiors:debian diff --git a/test_inferiors/README.md b/test_inferiors/README.md index fd99751..c9e4b4f 100644 --- a/test_inferiors/README.md +++ b/test_inferiors/README.md @@ -14,14 +14,15 @@ poetry run python inferior-sqlalchemy.py poetry run jupyter-lab ``` -These can be run in Docker. Build the image: +These can be run in Docker. Build the images: ```commandline -docker build . -t ivanyu/pyheap-test-inferiors +make docker-images ``` and run with one of the above commands: ```commandline -docker run --rm -ti ivanyu/pyheap-test-inferiors \ +docker run --rm -ti ivanyu/pyheap-test-inferiors:alpine \ poetry run python inferior-simple.py ``` +Use the `alpine` or `debian` tag. diff --git a/test_inferiors/alpine.Dockerfile b/test_inferiors/alpine.Dockerfile new file mode 100644 index 0000000..76b717a --- /dev/null +++ b/test_inferiors/alpine.Dockerfile @@ -0,0 +1,44 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM python:3.10.8-alpine3.16 + +RUN addgroup -g 10042 testgroup && adduser --uid 10123 testuser -g testgroup -D + +RUN set -eux; \ + apk add --no-cache --virtual .build-deps \ + gcc \ + g++ \ + linux-headers \ + libc-dev \ + openssl-dev + +ADD --chown=testuser:testgroup https://install.python-poetry.org /home/testuser/poetry-install.py + +USER testuser +RUN python3 /home/testuser/poetry-install.py + +COPY ./ /inferiors/ +WORKDIR /inferiors + +ENV PATH="/home/testuser/.local/bin:$PATH" + +RUN poetry install + +USER root +RUN set -eux; \ + apk del --no-network .build-deps + +USER testuser diff --git a/test_inferiors/debian.Dockerfile b/test_inferiors/debian.Dockerfile new file mode 100644 index 0000000..cdd234e --- /dev/null +++ b/test_inferiors/debian.Dockerfile @@ -0,0 +1,30 @@ +# +# Copyright 2022 Ivan Yurchenko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM python:3.10.8-bullseye + +RUN groupadd -g 10042 testgroup && useradd --uid 10123 testuser -g testgroup --create-home + +ADD --chown=testuser:testgroup https://install.python-poetry.org /home/testuser/poetry-install.py + +USER testuser +RUN python3 /home/testuser/poetry-install.py + +COPY ./ /inferiors/ +WORKDIR /inferiors + +ENV PATH="/home/testuser/.local/bin:$PATH" + +RUN poetry install