diff --git a/regularizepsf/fitter.py b/regularizepsf/fitter.py
index 1c6789d..e4fb94d 100644
--- a/regularizepsf/fitter.py
+++ b/regularizepsf/fitter.py
@@ -477,7 +477,7 @@ def average(self, corners: np.ndarray, patch_size: int, psf_size: int,  # noqa:
 
         for identifier, patch in self.patches.items():
             # Normalize the patch
-            patch = patch / np.max(patch)
+            patch = patch / patch[psf_size//2, psf_size//2]
 
             # Determine which average region it belongs to
             center_x = identifier.x + self.size // 2
diff --git a/tests/test_fitter.py b/tests/test_fitter.py
index 622c213..9afb92a 100644
--- a/tests/test_fitter.py
+++ b/tests/test_fitter.py
@@ -81,7 +81,7 @@ def test_coordinate_patch_average():
              })
     for patch in collection.values():
         # Make the normalization of each patch a no-op
-        patch[-1, -1] = 1
+        patch[5, 5] = 1
 
     averaged_collection = collection.average(
             np.array([[0, 0]]), 10, 10, mode='median')