diff --git a/qrcode/image/styles/colormasks.py b/qrcode/image/styles/colormasks.py
index 43186b2..495a9c7 100644
--- a/qrcode/image/styles/colormasks.py
+++ b/qrcode/image/styles/colormasks.py
@@ -29,22 +29,27 @@ def initialize(self, styledPilImage, image):
 
     def apply_mask(self, image):
         width, height = image.size
+        pixels = image.load()
+        fg_color_cache = {}
         for x in range(width):
             for y in range(height):
-                norm = self.extrap_color(
-                    self.back_color, self.paint_color, image.getpixel((x, y))
-                )
+                current_color = pixels[x, y]
+                if current_color == self.back_color:
+                    continue
+                if current_color in fg_color_cache:
+                    pixels[x, y] = fg_color_cache[current_color]
+                    continue
+                norm = self.extrap_color(self.back_color, self.paint_color, current_color)
                 if norm is not None:
-                    image.putpixel(
-                        (x, y),
-                        self.interp_color(
-                            self.get_bg_pixel(image, x, y),
-                            self.get_fg_pixel(image, x, y),
-                            norm,
-                        ),
+                    new_color = self.interp_color(
+                        self.get_bg_pixel(image, x, y),
+                        self.get_fg_pixel(image, x, y),
+                        norm
                     )
+                    pixels[x, y] = new_color
+                    fg_color_cache[current_color] = new_color
                 else:
-                    image.putpixel((x, y), self.get_bg_pixel(image, x, y))
+                    pixels[x, y] = self.get_bg_pixel(image, x, y)
 
     def get_fg_pixel(self, image, x, y):
         raise NotImplementedError("QRModuleDrawer.paint_fg_pixel")