Use Pillow to implement ImageDecoder (onnx#5555)

Use Pillow to replace opencv to implement ImageDecoder to remove
dependency on the heavy opencv. Code change made with help from Copilot
Chat.

### Motivation and Context

Fixes onnx#5545

---------

Signed-off-by: Justin Chu <[email protected]>
Co-authored-by: Chun-Wei Chen <[email protected]>

.github/workflows/release_win.yml

@@ -50,7 +50,8 @@ jobs:
         python -m pip install -q --upgrade pip
         cd onnx
         if ('${{ matrix.architecture }}' -eq 'x86') {
-          sed -i '' '/google-re2/d' requirements-release.txt
+          sed -i '' '/google-re2/d' requirements-reference.txt
+          sed -i '' '/Pillow/d' requirements-reference.txt
         }
         python -m pip install -q -r requirements-release.txt
 

docs/Operators.md

@@ -12138,7 +12138,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".bmp", "RGB")
+data, output = _generate_test_data("bmp", "RGB")
 expect(
     node,
     inputs=[data],
@@ -12161,7 +12161,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".jp2", "RGB")
+data, output = _generate_test_data("jpeg2000", "RGB")
 expect(
     node,
     inputs=[data],
@@ -12184,7 +12184,7 @@ node = onnx.helper.make_node(
     pixel_format="BGR",
 )
 
-data, output = generate_test_data(".jpg", "BGR")
+data, output = _generate_test_data("jpeg", "BGR")
 expect(
     node,
     inputs=[data],
@@ -12207,7 +12207,7 @@ node = onnx.helper.make_node(
     pixel_format="Grayscale",
 )
 
-data, output = generate_test_data(".jpg", "Grayscale")
+data, output = _generate_test_data("jpeg", "Grayscale")
 expect(
     node,
     inputs=[data],
@@ -12230,7 +12230,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".jpg", "RGB")
+data, output = _generate_test_data("jpeg", "RGB")
 expect(
     node,
     inputs=[data],
@@ -12253,7 +12253,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".png", "RGB")
+data, output = _generate_test_data("png", "RGB")
 expect(
     node,
     inputs=[data],
@@ -12276,7 +12276,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".pnm", "RGB")
+data, output = _generate_test_data("ppm", "RGB")
 expect(
     node,
     inputs=[data],
@@ -12299,7 +12299,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".tiff", "RGB")
+data, output = _generate_test_data("tiff", "RGB")
 expect(
     node,
     inputs=[data],
@@ -12322,7 +12322,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".webp", "RGB")
+data, output = _generate_test_data("webp", "RGB")
 expect(
     node,
     inputs=[data],

docs/TestCoverage.md

@@ -8194,7 +8194,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".bmp", "RGB")
+data, output = _generate_test_data("bmp", "RGB")
 expect(
     node,
     inputs=[data],
@@ -8215,7 +8215,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".jp2", "RGB")
+data, output = _generate_test_data("jpeg2000", "RGB")
 expect(
     node,
     inputs=[data],
@@ -8236,7 +8236,7 @@ node = onnx.helper.make_node(
     pixel_format="BGR",
 )
 
-data, output = generate_test_data(".jpg", "BGR")
+data, output = _generate_test_data("jpeg", "BGR")
 expect(
     node,
     inputs=[data],
@@ -8257,7 +8257,7 @@ node = onnx.helper.make_node(
     pixel_format="Grayscale",
 )
 
-data, output = generate_test_data(".jpg", "Grayscale")
+data, output = _generate_test_data("jpeg", "Grayscale")
 expect(
     node,
     inputs=[data],
@@ -8278,7 +8278,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".jpg", "RGB")
+data, output = _generate_test_data("jpeg", "RGB")
 expect(
     node,
     inputs=[data],
@@ -8299,7 +8299,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".png", "RGB")
+data, output = _generate_test_data("png", "RGB")
 expect(
     node,
     inputs=[data],
@@ -8320,7 +8320,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".pnm", "RGB")
+data, output = _generate_test_data("ppm", "RGB")
 expect(
     node,
     inputs=[data],
@@ -8341,7 +8341,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".tiff", "RGB")
+data, output = _generate_test_data("tiff", "RGB")
 expect(
     node,
     inputs=[data],
@@ -8362,7 +8362,7 @@ node = onnx.helper.make_node(
     pixel_format="RGB",
 )
 
-data, output = generate_test_data(".webp", "RGB")
+data, output = _generate_test_data("webp", "RGB")
 expect(
     node,
     inputs=[data],

onnx/backend/test/case/node/image_decoder.py

@@ -1,16 +1,19 @@
 # Copyright (c) ONNX Project Contributors
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import io
 
-import cv2
 import numpy as np
+import PIL.Image
 
 import onnx
 from onnx.backend.test.case.base import Base
 from onnx.backend.test.case.node import expect
 
 
-def generate_checkerboard(width, height, square_size):
+def generate_checkerboard(width: int, height: int, square_size: int) -> np.ndarray:
     # Create an empty RGB image
     image = np.zeros((height, width, 3), dtype=np.uint8)
 
@@ -39,22 +42,32 @@ def generate_checkerboard(width, height, square_size):
     return image
 
 
-def generate_test_data(extension, pixel_format="RGB", h=40, w=40, tile_sz=5):
-    data, output = None, None
+def _generate_test_data(
+    format_: str,
+    pixel_format: str = "RGB",
+    height: int = 32,
+    width: int = 32,
+    tile_sz: int = 5,
+) -> tuple[np.ndarray, np.ndarray]:
     np.random.seed(12345)
-    image = generate_checkerboard(h, w, tile_sz)
-    image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-    _, encoded_image = cv2.imencode(extension, image_bgr)
-    data = np.frombuffer(encoded_image, dtype=np.uint8)
+    image = generate_checkerboard(height, width, tile_sz)
+    image_pil = PIL.Image.fromarray(image)
+    with io.BytesIO() as f:
+        image_pil.save(f, format=format_)
+        data = f.getvalue()
+        data_array = np.frombuffer(data, dtype=np.uint8)
     if pixel_format == "BGR":
-        output = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        output_pil = PIL.Image.open(io.BytesIO(data))
+        output = np.array(output_pil)[:, :, ::-1]
     elif pixel_format == "RGB":
-        output_bgr = cv2.imdecode(data, cv2.IMREAD_COLOR)
-        output = cv2.cvtColor(output_bgr, cv2.COLOR_BGR2RGB)
+        output_pil = PIL.Image.open(io.BytesIO(data))
+        output = np.array(output_pil)
     elif pixel_format == "Grayscale":
-        output = cv2.imdecode(data, cv2.IMREAD_GRAYSCALE)
-        output = np.expand_dims(output, axis=2)  # (H, W) to (H, W, 1)
-    return data, output
+        output_pil = PIL.Image.open(io.BytesIO(data)).convert("L")
+        output = np.array(output_pil)[:, :, np.newaxis]
+    else:
+        raise ValueError(f"Unsupported pixel format: {pixel_format}")
+    return data_array, output
 
 
 class ImageDecoder(Base):
@@ -67,7 +80,7 @@ def export_image_decoder_decode_jpeg_rgb() -> None:
             pixel_format="RGB",
         )
 
-        data, output = generate_test_data(".jpg", "RGB")
+        data, output = _generate_test_data("jpeg", "RGB")
         expect(
             node,
             inputs=[data],
@@ -84,7 +97,7 @@ def export_image_decoder_decode_jpeg_grayscale() -> None:
             pixel_format="Grayscale",
         )
 
-        data, output = generate_test_data(".jpg", "Grayscale")
+        data, output = _generate_test_data("jpeg", "Grayscale")
         expect(
             node,
             inputs=[data],
@@ -101,7 +114,7 @@ def export_image_decoder_decode_jpeg_bgr() -> None:
             pixel_format="BGR",
         )
 
-        data, output = generate_test_data(".jpg", "BGR")
+        data, output = _generate_test_data("jpeg", "BGR")
         expect(
             node,
             inputs=[data],
@@ -118,7 +131,7 @@ def export_image_decoder_decode_jpeg2k_rgb() -> None:
             pixel_format="RGB",
         )
 
-        data, output = generate_test_data(".jp2", "RGB")
+        data, output = _generate_test_data("jpeg2000", "RGB")
         expect(
             node,
             inputs=[data],
@@ -135,7 +148,7 @@ def export_image_decoder_decode_bmp_rgb() -> None:
             pixel_format="RGB",
         )
 
-        data, output = generate_test_data(".bmp", "RGB")
+        data, output = _generate_test_data("bmp", "RGB")
         expect(
             node,
             inputs=[data],
@@ -152,7 +165,7 @@ def export_image_decoder_decode_png_rgb() -> None:
             pixel_format="RGB",
         )
 
-        data, output = generate_test_data(".png", "RGB")
+        data, output = _generate_test_data("png", "RGB")
         expect(
             node,
             inputs=[data],
@@ -169,7 +182,7 @@ def export_image_decoder_decode_tiff_rgb() -> None:
             pixel_format="RGB",
         )
 
-        data, output = generate_test_data(".tiff", "RGB")
+        data, output = _generate_test_data("tiff", "RGB")
         expect(
             node,
             inputs=[data],
@@ -186,7 +199,7 @@ def export_image_decoder_decode_webp_rgb() -> None:
             pixel_format="RGB",
         )
 
-        data, output = generate_test_data(".webp", "RGB")
+        data, output = _generate_test_data("webp", "RGB")
         expect(
             node,
             inputs=[data],
@@ -203,7 +216,7 @@ def export_image_decoder_decode_pnm_rgb() -> None:
             pixel_format="RGB",
         )
 
-        data, output = generate_test_data(".pnm", "RGB")
+        data, output = _generate_test_data("ppm", "RGB")
         expect(
             node,
             inputs=[data],

onnx/backend/test/data/node/test_image_decoder_decode_jpeg_grayscale/test_data_set_0/output_0.pb

@@ -1 +1,10 @@
-((
BoutputJ���������������������:9999srsqu65538OPLRM��������������������89:98vqtwl74753ONPPP��������������������7:998upvrv27655ONOMN��������������������89:98wrptt43248NQOON��������������������;:888rwtrs38:51PPMNQ^\b`[iba`_�����bdghd94598IMKJOxzz�����`]a``abegc�����ecaag39968OMLNJz�~|y�����_`]_`cbcaa�����dedf`<6876JKNLJ{{||�����__^_^ebccc�����ddcdb97767NLLJK}|||}�����_]b_^dcddb�����aedef57876KLLMM}{{{}�����}}}~�����b]_``eb__eDJJHH���������������~y~x�����^]c\\acdcaIGFGG���������������|z~~�����_^_^acabaaIIHIJ���������������|~{|�����````\acaaeDHHGE���������������|�{|~�����]_^]caacc^KGFIJ������������������������b`cc_gfgeg������������������������������`abbbgfgfe�����������������������������bcca`fegdh������������������������������a`bcbgfefg����������������������������cabb_eehdf���������–��������������nkkjncdddfqpoqqLLOMM�����SROQT����������mknjleedccooqopNOMOM�����NQRSN����������kkmnkdbdeepqpqoOMNMM�����PSORQ����������mlkljfdddcpoporLONOO�����QQRPS����������hqimncccfaroqpoNNLNN�����QPQPP�����bdabf��½��������������������Ϋ����QVTVSbbeaa���Ŀ�������������������ͬ����V[QT[eafdc����Š������������������ʮ����XWWWQbc_bb������������������������Ͱ����VUTZWcfdbc���Ŀ�������������������ͫ����TXUVU<8=:>����։����^]\^_�����=<>?=�����spstr<;9=8����Ў����__\`[�����=??=@�����qrrrr;;:=;����ԇ����^^_^]�����==<>@�����qsrsq9::<:����ԋ����]]\^\�����>?D8>�����qsvpr=>;;9����э����``^`^�����=>;A@�����qqmst)+*,/����ற��������TWVVXuursrYZ[YY�����--,,,����ݮ���������YWWVVssrwuZ[ZXZ�����+++,+����޲���������WTWXWtvust[YW]Y�����,+,-+����య��������UYYTVwtrsuYZ]X]�����-+,-*����ޯ���������YWTYXrsuusY[YW[�����
+  
BoutputJ���������������������0?3=-|pskw��������������������;5<=Azgvws��������������������<388:vnuzr	��������������������;E4='s|oqu		��������������������D5(=Lpwfls7454.VTWW[XY]XZimVYp��Ĭ�mbbmj8566-PKMPX\_d_]hZavf�����f_dfb31692UPQGSZ_a^ZcegY\�����`mgY`5863:IQOKQbdXZbedj^`�����fh\jW/32/7GOMQF[`^d\f^cbp�����jmcma6:96=LSQTO�x�����̾�}y{������
+69849GLIGC�|ry������yz�w�����
+6;:6<IOK[L|}�y�����u��t�����2674;INKGHt|�}z�����|}~�y�����6;:6<GKGO]v{~s�����Ńzt�������`c`Z]ega`\IDNBM���������������_____`bbb^KEJNC���������������
]]^_`accgePHFID���������������
]^_`bcde^_JFCIK���������������\]^^___^deMNKQS������������������������}]`[\Wndfec�������½�������~��_hcd_cfhhf������ſ���������~Zf`dfcegfe��������������������_i]`gbegfe�����¸���
+��������}cbebi^iddq�������������������������nifddfgebavlpm}
+���������������nsuuphcce^uoums
+
���������������heikhb_bfesjoim���������������jmmmjgdddfqjtof���������������tmkhggecacnmzuf		
+	
+
+
+


		

onnx/reference/ops/op_image_decoder.py

@@ -1,35 +1,33 @@
 # Copyright (c) ONNX Project Contributors
 
 # SPDX-License-Identifier: Apache-2.0
-# pylint: disable=C0123,C3001,R0912,R0913,R0914,R1730,W0221,W0613
+
+from __future__ import annotations
+
+import io
 
 import numpy as np
 
 from onnx.reference.op_run import OpRun
 
 
 class ImageDecoder(OpRun):
-    def _run(  # type: ignore
-        self,
-        encoded,
-        pixel_format="RGB",
-    ):
+    def _run(self, encoded: np.ndarray, pixel_format="RGB") -> tuple[np.ndarray]:  # type: ignore
         try:
-            # pylint: disable=import-outside-toplevel`
-            import cv2
+            import PIL.Image  # pylint: disable=import-outside-toplevel
         except ImportError as e:
             raise ImportError(
-                "opencv-python must be installed to use the reference implementation of the ImageDecoder operator"
+                "Pillow must be installed to use the reference implementation of the ImageDecoder operator"
             ) from e
-        decoded = None
+        img = PIL.Image.open(io.BytesIO(encoded.tobytes()))
         if pixel_format == "BGR":
-            decoded = cv2.imdecode(encoded, cv2.IMREAD_COLOR)
+            decoded = np.array(img)[:, :, ::-1]
         elif pixel_format == "RGB":
-            decoded = cv2.imdecode(encoded, cv2.IMREAD_COLOR)
-            decoded = cv2.cvtColor(decoded, cv2.COLOR_BGR2RGB)
+            decoded = np.array(img)
         elif pixel_format == "Grayscale":
-            decoded = cv2.imdecode(encoded, cv2.IMREAD_GRAYSCALE)
+            img = img.convert("L")
+            decoded = np.array(img)
             decoded = np.expand_dims(decoded, axis=2)  # (H, W) to (H, W, 1)
         else:
-            raise RuntimeError(f"pixel_format={pixel_format!r} is not supported.")
+            raise ValueError(f"pixel_format={pixel_format!r} is not supported.")
         return (decoded,)