Merge pull request #109 from obiwac/ep13b

Episode 13b: Performance improvements

.github/workflows/ep13-lsp.yml

@@ -14,13 +14,14 @@ jobs:
         with:
           python-version: "3.11"
       - uses: abatilo/actions-poetry@v2
-      - name: Install dependencies
-        run: poetry install --no-root --with dev
+      - name: Install dependencies and build Cython extension
+        run: poetry install --no-root
+      - name: Rebuild the project
+        run: poetry build
       - name: Cache venv created by poetry (configured to be in '.venv')
         uses: actions/cache@v3
         with:
           path: ./.venv
           key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
       - name: Run pyright
-        run: |
-          poetry run pyright
+        run: poetry run pyright

.gitignore

@@ -4,3 +4,12 @@
 .vscode
 *.ogg
 *.sw[nop]
+*.so
+*.c
+*.so
+*.html
+*.cache
+dist
+build
+*.egg-info
+*.prof

episode-13/README.md

@@ -7,4 +7,27 @@ Clock on the thumbnail below to watch the video:
 This episode is split into two parts:
 
 - EP13a: The code has gotten a little crusty. This episode restructures the codebase, adds a `pyproject.toml` file and switches to Poetry for better dependency management, and adds a formatter and a linter as well as a CI setup to check all this.
-- EP13b: Loading big save files took a long-ass time before this. This episode covers profiling and optimization techniques and rewrites a lot of the chunk loading code in Cython to speed it up dramatically.
+- EP13b: Loading big save files took a long-ass time before this. This episode covers profiling and optimization techniques and rewrites a lot of the chunk loading code in Cython to speed it up dramatically. It also covers some frame time improvements in preparation for adding mobs.
+
+## Installing dependencies
+
+Using [Poetry](https://python-poetry.org/):
+
+```console
+poetry install
+```
+
+This will also build the Cython extensions.
+If you ever need to rebuild them, you can do:
+
+```console
+poetry build
+```
+
+## Running
+
+Again, using Poetry:
+
+```console
+poetry run python mcpy.py
+```

episode-13/build.py

@@ -0,0 +1,28 @@
+from setuptools.command.build_ext import build_ext
+from Cython.Build import cythonize
+import Cython.Compiler.Options
+
+Cython.Compiler.Options.cimport_from_pyx = True  # needed?
+
+
+class BuildExt(build_ext):
+	def build_extension(self, ext):
+		self.inplace = True  # Important or the LSP won't have access to the compiled files.
+		super().build_extension(ext)
+
+
+def build(setup_kwargs):
+	ext_modules = cythonize(
+		[
+			"src/chunk/__init__.pyx",
+			"src/chunk/chunk.pyx",
+			"src/chunk/subchunk.pyx",
+		],
+		compiler_directives={
+			"language_level": 3,
+			"profile": True,
+		},
+		annotate=True,
+	)
+
+	setup_kwargs.update({"ext_modules": ext_modules, "cmdclass": {"build_ext": BuildExt}})

episode-13/main.py → episode-13/mcpy.py

@@ -1,5 +1,6 @@
 import math
 import random
+from cProfile import Profile
 import pyglet
 
 pyglet.options["shadow_window"] = False
@@ -50,6 +51,32 @@ def update(self, delta_time):
 
 		self.player.update(delta_time)
 
+		# Load the closest chunk which hasn't been loaded yet.
+
+		x, y, z = self.player.position
+		closest_chunk = None
+		min_distance = math.inf
+
+		for chunk_pos, chunk in self.world.chunks.items():
+			if chunk.loaded:
+				continue
+
+			cx, cy, cz = chunk_pos
+
+			cx *= CHUNK_WIDTH
+			cy *= CHUNK_HEIGHT
+			cz *= CHUNK_LENGTH
+
+			dist = (cx - x) ** 2 + (cy - y) ** 2 + (cz - z) ** 2
+
+			if dist < min_distance:
+				min_distance = dist
+				closest_chunk = chunk
+
+		if closest_chunk is not None:
+			closest_chunk.update_subchunk_meshes()
+			closest_chunk.update_mesh()
+
 	def on_draw(self):
 		self.player.update_matrices()
 
@@ -220,6 +247,38 @@ def run(self):
 		pyglet.app.run()
 
 
+def sample_initial_loading_time():
+	with Profile() as profiler:
+		Game()
+		profiler.create_stats()
+		profiler.dump_stats("stats.prof")
+
+		for k in profiler.stats.keys():
+			# file line name
+			file, _, name = k
+
+			if "world.py" in file and name == "__init__":
+				# ? ncalls time cumtime parent
+				_, _, _, cumtime, _ = profiler.stats[k]
+				break
+
+		else:
+			raise Exception("Couldn't find work init stats!")
+
+		return cumtime
+
+
+def benchmark_initial_loading_time():
+	n = 10
+	samples = [sample_initial_loading_time() for _ in range(n)]
+	mean = sum(samples) / n
+
+	print(mean)
+	exit()
+
+
 if __name__ == "__main__":
+	# benchmark_initial_loading_time()
+
 	game = Game()
 	game.run()

episode-13/pyproject.toml

@@ -14,15 +14,24 @@ authors = [
 ]
 readme = "README.md"
 
+[build-system]
+requires = ["poetry-core", "setuptools", "cython"]
+
+[tool.poetry.build]
+script = "build.py"
+generate-setup-file = true
+
 [tool.poetry.dependencies]
 python = "^3.10"
 pyglet = "^2.0.16"
 nbtlib = "^2.0.4"
 base36 = "^0.1.1"
+pyglm = "^2.7.1"
 
 [tool.poetry.group.dev.dependencies]
 ruff = "^0.5.5"
 pyright = "^1.1.374"
+cython = "^3.0.11"
 
 [tool.pyright]
 exclude = [".venv"]
@@ -34,6 +43,6 @@ venv = ".venv"
 reportIndexIssue = false
 
 # From https://github.com/obiwac/python-minecraft-clone/pull/107:
-# F405 * may be undefined, or defined from star imports: These are indeed defined from star imports. I guess we could import all the symbols in '__all__' explicitly, but if there's a mistake here and it causes a runtime error, that's not the end of the world.
+# F405 X may be undefined, or defined from star imports: These are indeed defined from star imports. I guess we could import all the symbols in '__all__' explicitly, but if there's a mistake here and it causes a runtime error, that's not the end of the world.
 
 ignore = ["models/__init__.py"]

episode-13/src/chunk/__init__.py

@@ -0,0 +1,11 @@
+from src.chunk.common import CHUNK_HEIGHT, CHUNK_LENGTH, CHUNK_WIDTH
+from src.chunk.common import SUBCHUNK_HEIGHT, SUBCHUNK_LENGTH, SUBCHUNK_WIDTH
+
+__all__ = [
+	"CHUNK_WIDTH",
+	"CHUNK_HEIGHT",
+	"CHUNK_LENGTH",
+	"SUBCHUNK_WIDTH",
+	"SUBCHUNK_HEIGHT",
+	"SUBCHUNK_LENGTH",
+]

episode-13/src/chunk/__init__.pyx

@@ -0,0 +1,60 @@
+from src.chunk.common import CHUNK_WIDTH, CHUNK_HEIGHT, CHUNK_LENGTH
+from src.chunk.common import SUBCHUNK_WIDTH, SUBCHUNK_HEIGHT, SUBCHUNK_LENGTH
+
+from libc.stdlib cimport malloc, free
+from libc.string cimport memset
+from libc.stdint cimport uint8_t, uint32_t
+
+cdef int C_CHUNK_WIDTH = CHUNK_WIDTH
+cdef int C_CHUNK_HEIGHT = CHUNK_HEIGHT
+cdef int C_CHUNK_LENGTH = CHUNK_LENGTH
+
+cdef int C_SUBCHUNK_WIDTH = SUBCHUNK_WIDTH
+cdef int C_SUBCHUNK_HEIGHT = SUBCHUNK_HEIGHT
+cdef int C_SUBCHUNK_LENGTH = SUBCHUNK_LENGTH
+
+cdef class CSubchunk:
+	cdef size_t data_count
+	cdef float* data
+
+	cdef size_t index_count
+	cdef uint32_t* indices
+
+	def __init__(self):
+		self.data_count = 0
+		self.index_count = 0
+
+cdef class CChunk:
+	cdef size_t data_count
+	cdef float* data
+
+	cdef size_t index_count
+	cdef int* indices
+
+	cdef size_t size
+	cdef uint8_t* blocks
+
+	def __init__(self):
+		self.size = CHUNK_WIDTH * CHUNK_HEIGHT * CHUNK_LENGTH * sizeof(self.blocks[0])
+		self.blocks = <uint8_t*>malloc(self.size)
+		memset(self.blocks, 0, self.size)
+
+	def __del__(self):
+		free(self.blocks)
+
+	@property
+	def index_count(self):
+		return self.index_count
+
+	def get_blocks(self, i):
+		return self.blocks[i]
+
+	def set_blocks(self, i, val):
+		self.blocks[i] = val
+
+	def copy_blocks(self, blocks):
+		cdef int i
+		cdef int length = len(blocks)
+
+		for i in range(length):
+			self.blocks[i] = blocks[i]