dumper: ensure dumping can work without having GDB in the container

The main theme of this commit is to ensure the dumper works fine without GDB installed in the target namespace/container.
Besides that, the commit:
1. Introduces integration testing with three more Linux distributions--Fedora, Debian, Ubuntu--in addition to Alpine Linux.
2. Does some refactoring on the dumper side.

Closes #200
Closes #209

Makefile

@@ -18,43 +18,43 @@
 clean:
 	(cd pyheap && $(MAKE) clean)
 
-.PHONY: integration_tests
-integration_tests: integration_tests_3_8 integration_tests_3_9 integration_tests_3_10 integration_tests_3_11
+.PHONY: integration-tests
+integration-tests: integration-tests-3-8 integration-tests-3-9 integration-tests-3-10 integration-tests-3-11
 
 pyheap/dist/pyheap_dump.pyz:
 	(cd pyheap && $(MAKE) dist)
 
-.PHONY: integration_tests_3_8
-integration_tests_3_8: pyheap/dist/pyheap_dump.pyz
+.PHONY: integration-tests-3-8
+integration-tests-3-8: pyheap/dist/pyheap_dump.pyz
 	(cd integration_tests && \
-	  $(MAKE) test_target_docker_image_3_8 && \
+	  $(MAKE) test-target-docker-images-3-8 && \
 	  PYENV_VERSION=3.8 poetry env use python && \
 	  poetry run pip install -e ../pyheap-ui/ && \
 	  poetry install && \
 	  poetry run pytest -vv ./*.py)
 
-.PHONY: integration_tests_3_9
-integration_tests_3_9: pyheap/dist/pyheap_dump.pyz
+.PHONY: integration-tests-3-9
+integration-tests-3-9: pyheap/dist/pyheap_dump.pyz
 	(cd integration_tests && \
-	  $(MAKE) test_target_docker_image_3_9 && \
+	  $(MAKE) test-target-docker-images-3-9 && \
 	  PYENV_VERSION=3.9 poetry env use python && \
 	  poetry run pip install -e ../pyheap-ui/ && \
 	  poetry install && \
 	  poetry run pytest -vv ./*.py)
 
-.PHONY: integration_tests_3_10
-integration_tests_3_10: pyheap/dist/pyheap_dump.pyz
+.PHONY: integration-tests-3-10
+integration-tests-3-10: pyheap/dist/pyheap_dump.pyz
 	(cd integration_tests && \
-	  $(MAKE) test_target_docker_image_3_10 && \
+	  $(MAKE) test-target-docker-images-3-10 && \
 	  PYENV_VERSION=3.10 poetry env use python && \
 	  poetry run pip install -e ../pyheap-ui/ && \
 	  poetry install && \
 	  poetry run pytest -vv ./*.py)
 
-.PHONY: integration_tests_3_11
-integration_tests_3_11: pyheap/dist/pyheap_dump.pyz
+.PHONY: integration-tests-3-11
+integration-tests-3-11: pyheap/dist/pyheap_dump.pyz
 	(cd integration_tests && \
-	  $(MAKE) test_target_docker_image_3_11 && \
+	  $(MAKE) test-target-docker-images-3-11 && \
 	  PYENV_VERSION=3.11 poetry env use python && \
 	  poetry run pip install -e ../pyheap-ui/ && \
 	  poetry install && \

README.md

@@ -3,16 +3,27 @@
 A heap dumper and analyzer for CPython based on GDB.
 
 The product consists of two parts:
-1. The zero-dependency dumper script.
+1. The dumper which uses GDB.
 2. The Flask-based UI for heap dump visualization.
 
+## Requirements
+
+The dumper needs the following:
+1. GDB must be installed where the dumper runs (e.g. on the machine host), but is not needed near a target process (e.g. in a container).
+2. CPython 3.8 - 3.11.
+3. Docker CLI for working with Docker containers directly (e.g. calling `docker inspect`).
+
 ## Compatibility
 
-The dumper is compatible with a target process running on:
-- CPython 3.8;
-- CPython 3.9;
-- CPython 3.10;
-- CPython 3.11.
+**Only Linux** is supported at the moment.
+
+The dumper is compatible with a target process running on CPython 3.8 - 3.11.
+
+The target process were tested in the following OSes:
+- Alpine Linux;
+- Ubuntu;
+- Fedora;
+- Debian.
 
 Some popular libraries were tested:
 - Django;
@@ -55,8 +66,6 @@ $ sudo python3 pyheap_dump.pyz --docker-container <container_name> --file heap.p
 
 If it's not the root process in the container, or you work with another container system (e.g. systemd-nspawn) or just generic Linux namespaces, you need to find the target PID. Please mind that this must be the PID from the dumper point of view: processes in namespaces can have their own PID numbers. For example, if you're about to run the dumper on a Linux host and the target process is running in a container, check the process list with `ps` or `top` on the host. Use `--pid/-p` for the dumper.
 
-Make sure the GDB executable is available in the target mount namespace. If the target is in a Docker container, you most likely need to install GDB inside it (please note this can be done in a running container as well).
-
 If the target process is running under a different user (normal for Docker), you need to use `sudo` with `python3 pyheap_dump.pyz ...`.
 
 PyHeap dumper will automatically transfer the heap file from the target namespace to the specified location.
@@ -169,7 +178,7 @@ Currently, the dumper sees objects traced by the CPython garbage collector and t
 
 Integration tests run on CI. However, end-to-end tests that use the real GDB cannot be run in GitHub Actions. You can run them locally using
 ```bash
-make clean integration_tests
+make clean integration-tests
 ```
 
 You need [pyenv](https://github.com/pyenv/pyenv) with Python 3.8, 3.9, 3.10, and 3.11 installed and [Poetry](https://python-poetry.org/).

integration_tests/Makefile

@@ -18,29 +18,41 @@
 clean:
 	rm e2e_docker/inferior-simple.py
 
-.PHONY: test_target_docker_image_3_8
-test_target_docker_image_3_8: e2e_docker/inferior-simple.py
+define build_image
 	docker build e2e_docker \
-		--build-arg BASE_VERSION="3.8.15-bullseye" \
-		--tag ivanyu/pyheap-e2e-test-target:3.8
+		-f "e2e_docker/$2.Dockerfile" \
+		--build-arg BASE_IMAGE_VERSION="$3" \
+		--build-arg PYHEAP_PYTHON_VERSION="$1" \
+		--tag "ivanyu/pyheap-e2e-test-target:$2-$1"
+endef
 
-.PHONY: test_target_docker_image_3_9
-test_target_docker_image_3_9: e2e_docker/inferior-simple.py
-	docker build e2e_docker \
-		--build-arg BASE_VERSION="3.9.15-bullseye" \
-		--tag ivanyu/pyheap-e2e-test-target:3.9
+.PHONY: test-target-docker-images-3-8
+test-target-docker-images-3-8: e2e_docker/inferior-simple.py
+	$(call build_image,3.8,alpine,3.8.15-alpine3.16)
+	$(call build_image,3.8,debian,3.8.15-slim-bullseye)
+	$(call build_image,3.8,ubuntu,22.04)
+	$(call build_image,3.8,fedora,36)
 
-.PHONY: test_target_docker_image_3_10
-test_target_docker_image_3_10: e2e_docker/inferior-simple.py
-	docker build e2e_docker \
-		--build-arg BASE_VERSION="3.10.8-bullseye" \
-		--tag ivanyu/pyheap-e2e-test-target:3.10
+.PHONY: test-target-docker-images-3-9
+test-target-docker-images-3-9: e2e_docker/inferior-simple.py
+	$(call build_image,3.9,alpine,3.9.15-alpine3.16)
+	$(call build_image,3.9,debian,3.9.15-slim-bullseye)
+	$(call build_image,3.9,ubuntu,22.04)
+	$(call build_image,3.9,fedora,36)
 
-.PHONY: test_target_docker_image_3_11
-test_target_docker_image_3_11: e2e_docker/inferior-simple.py
-	docker build e2e_docker \
-		--build-arg BASE_VERSION="3.11.0-bullseye" \
-		--tag ivanyu/pyheap-e2e-test-target:3.11
+.PHONY: test-target-docker-images-3-10
+test-target-docker-images-3-10: e2e_docker/inferior-simple.py
+	$(call build_image,3.10,alpine,3.10.8-alpine3.16)
+	$(call build_image,3.10,debian,3.10.8-slim-bullseye)
+	$(call build_image,3.10,ubuntu,22.04)
+	$(call build_image,3.10,fedora,36)
+
+.PHONY: test-target-docker-images-3-11
+test-target-docker-images-3-11: e2e_docker/inferior-simple.py
+	$(call build_image,3.11,alpine,3.11.0-alpine3.16)
+	$(call build_image,3.11,debian,3.11.0-slim-bullseye)
+	$(call build_image,3.11,ubuntu,22.04)
+	$(call build_image,3.11,fedora,36)
 
 e2e_docker/inferior-simple.py:
 	cp ../test_inferiors/inferior-simple.py $@

integration_tests/e2e_docker/.gitignore

@@ -1,2 +1,16 @@
-*
-!Dockerfile
+#
+# Copyright 2022 Ivan Yurchenko
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+inferior-simple.py

integration_tests/e2e_docker/alpine.Dockerfile

@@ -0,0 +1,24 @@
+#
+# Copyright 2022 Ivan Yurchenko
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ARG BASE_IMAGE_VERSION
+FROM python:$BASE_IMAGE_VERSION
+
+ARG PYHEAP_PYTHON_VERSION
+
+COPY inferior-simple.py /inferior-simple.py
+
+RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python
+CMD ["/test-python", "/inferior-simple.py"]

integration_tests/e2e_docker/debian.Dockerfile

@@ -0,0 +1,24 @@
+#
+# Copyright 2022 Ivan Yurchenko
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ARG BASE_IMAGE_VERSION
+FROM python:$BASE_IMAGE_VERSION
+
+ARG PYHEAP_PYTHON_VERSION
+
+COPY inferior-simple.py /inferior-simple.py
+
+RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python
+CMD ["/test-python", "/inferior-simple.py"]

integration_tests/e2e_docker/fedora.Dockerfile

@@ -0,0 +1,29 @@
+#
+# Copyright 2022 Ivan Yurchenko
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ARG BASE_IMAGE_VERSION
+FROM fedora:$BASE_IMAGE_VERSION
+
+ARG PYHEAP_PYTHON_VERSION
+
+RUN dnf -y update && dnf install -y \
+        "python${PYHEAP_PYTHON_VERSION}" \
+        which \
+    && dnf clean all
+
+COPY inferior-simple.py /inferior-simple.py
+
+RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python
+CMD ["/test-python", "/inferior-simple.py"]

integration_tests/e2e_docker/ubuntu.Dockerfile

@@ -0,0 +1,32 @@
+#
+# Copyright 2022 Ivan Yurchenko
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ARG BASE_IMAGE_VERSION
+FROM ubuntu:$BASE_IMAGE_VERSION
+
+ARG PYHEAP_PYTHON_VERSION
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+RUN apt-get update \
+    && apt-get install -y software-properties-common \
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get install -y "python${PYHEAP_PYTHON_VERSION}" \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY inferior-simple.py /inferior-simple.py
+
+RUN ln -s $(which "python${PYHEAP_PYTHON_VERSION}") /test-python
+CMD ["/test-python", "/inferior-simple.py"]

integration_tests/manual_test_e2e.py

@@ -13,23 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import json
 import mmap
 import os
 import subprocess
 import sys
 import time
 from contextlib import contextmanager, closing
-from typing import Iterator, Union
+from typing import Iterator, Union, Optional
 import pytest
 from _pytest.tmpdir import TempPathFactory
 from pyheap_ui.heap_reader import HeapReader
 
 
-@pytest.mark.parametrize("docker", [False, True])
-def test_e2e(docker: bool, test_heap_path: str) -> None:
-    with _inferior_process(docker) as ip_pid_or_container, _dumper_process(
-        test_heap_path, ip_pid_or_container, docker
+@pytest.mark.parametrize("docker_base", ["alpine", "debian", "ubuntu", "fedora", None])
+def test_e2e(docker_base: Optional[str], test_heap_path: str) -> None:
+    is_docker = docker_base is not None
+    with _inferior_process(docker_base) as ip_pid_or_container, _dumper_process(
+        test_heap_path, ip_pid_or_container, is_docker
     ) as dp:
         print(f"Inferior process/container {ip_pid_or_container}")
         print(f"Dumper process {dp.pid}")
@@ -72,15 +72,15 @@ def _inferior_process_plain() -> Iterator[int]:
 
 
 @contextmanager
-def _inferior_process_docker() -> Iterator[str]:
+def _inferior_process_docker(docker_base: str) -> Iterator[str]:
     python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
     docker_proc = subprocess.run(
         [
             "docker",
             "run",
             "--rm",
             "--detach",
-            f"ivanyu/pyheap-e2e-test-target:{python_version}",
+            f"ivanyu/pyheap-e2e-test-target:{docker_base}-{python_version}",
         ],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
@@ -103,9 +103,9 @@ def _inferior_process_docker() -> Iterator[str]:
 
 
 @contextmanager
-def _inferior_process(docker: bool) -> Iterator[Union[int, str]]:
-    if docker:
-        with _inferior_process_docker() as r:
+def _inferior_process(docker_base: Optional[str]) -> Iterator[Union[int, str]]:
+    if docker_base is not None:
+        with _inferior_process_docker(docker_base) as r:
             yield r
     else:
         with _inferior_process_plain() as r: