diff --git a/docs/Changelog.md b/docs/Changelog.md index 3bee59ad260..97b62b5f7d0 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -24092,6 +24092,66 @@ This version of the operator has been available since version 20 of the default
Constrain grid types to float tensors.
+### **ImageDecoder-20** + + Loads and decodes and image from a file. If it can't decode for any reason (e.g. corrupted encoded + stream, invalid format, it will return an empty matrix). + The following image formats are supported: + * BMP + * JPEG (note: Lossless JPEG support is optional) + * JPEG2000 + * TIFF + * PNG + * WebP + * Portable image format (PBM, PGM, PPM, PXM, PNM) + Decoded images follow a channel-last layout: (Height, Width, Channels). + **JPEG chroma upsampling method:** + When upsampling the chroma components by a factor of 2, the pixels are linearly interpolated so that the + centers of the output pixels are 1/4 and 3/4 of the way between input pixel centers. + When rounding, 0.5 is rounded down and up at alternative pixels locations to prevent bias towards + larger values (ordered dither pattern). + Considering adjacent input pixels A, B, and C, B is upsampled to pixels B0 and B1 so that + ``` + B0 = round_half_down((1/4) * A + (3/4) * B) + B1 = round_half_up((3/4) * B + (1/4) * C) + ``` + This method, is the default chroma upsampling method in the well-established libjpeg-turbo library, + also referred as "smooth" or "fancy" upsampling. + +#### Version + +This version of the operator has been available since version 20 of the default ONNX operator set. + +#### Attributes + +
+
pixel_format : string (default is RGB)
+
Pixel format. Can be one of "RGB", "BGR", or "Grayscale".
+
+ +#### Inputs + +
+
encoded_stream (non-differentiable) : T1
+
Encoded stream
+
+ +#### Outputs + +
+
image (non-differentiable) : T2
+
Decoded image
+
+ +#### Type Constraints + +
+
T1 : tensor(uint8)
+
Constrain input types to 8-bit unsigned integer tensor.
+
T2 : tensor(uint8)
+
Constrain output types to 8-bit unsigned integer tensor.
+
+ ### **RegexFullMatch-20** RegexFullMatch performs a full regex match on each element of the input tensor. If an element fully matches the regex pattern specified as an attribute, the corresponding element in the output is True and it is False otherwise. [RE2](https://github.com/google/re2/wiki/Syntax) regex syntax is used. diff --git a/docs/Operators.md b/docs/Operators.md index 49a72bfce1f..197b95bc330 100644 --- a/docs/Operators.md +++ b/docs/Operators.md @@ -71,6 +71,7 @@ For an operator input/output's differentiability, it can be differentiable, |Hardmax|13, 11, 1| |Identity|19, 16, 14, 13, 1| |If|19, 16, 13, 11, 1| +|ImageDecoder|20| |InstanceNormalization|6, 1| |IsInf|10| |IsNaN|13, 9| @@ -12063,6 +12064,276 @@ expect( +### **ImageDecoder** + + Loads and decodes and image from a file. If it can't decode for any reason (e.g. corrupted encoded + stream, invalid format, it will return an empty matrix). + The following image formats are supported: + * BMP + * JPEG (note: Lossless JPEG support is optional) + * JPEG2000 + * TIFF + * PNG + * WebP + * Portable image format (PBM, PGM, PPM, PXM, PNM) + Decoded images follow a channel-last layout: (Height, Width, Channels). + **JPEG chroma upsampling method:** + When upsampling the chroma components by a factor of 2, the pixels are linearly interpolated so that the + centers of the output pixels are 1/4 and 3/4 of the way between input pixel centers. + When rounding, 0.5 is rounded down and up at alternative pixels locations to prevent bias towards + larger values (ordered dither pattern). + Considering adjacent input pixels A, B, and C, B is upsampled to pixels B0 and B1 so that + ``` + B0 = round_half_down((1/4) * A + (3/4) * B) + B1 = round_half_up((3/4) * B + (1/4) * C) + ``` + This method, is the default chroma upsampling method in the well-established libjpeg-turbo library, + also referred as "smooth" or "fancy" upsampling. + +#### Version + +This version of the operator has been available since version 20 of the default ONNX operator set. + +#### Attributes + +
+
pixel_format : string (default is RGB)
+
Pixel format. Can be one of "RGB", "BGR", or "Grayscale".
+
+ +#### Inputs + +
+
encoded_stream (non-differentiable) : T1
+
Encoded stream
+
+ +#### Outputs + +
+
image (non-differentiable) : T2
+
Decoded image
+
+ +#### Type Constraints + +
+
T1 : tensor(uint8)
+
Constrain input types to 8-bit unsigned integer tensor.
+
T2 : tensor(uint8)
+
Constrain output types to 8-bit unsigned integer tensor.
+
+ + +#### Examples + +
+image_decoder_decode_bmp_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".bmp", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_bmp_rgb", +) +``` + +
+ + +
+image_decoder_decode_jpeg2k_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".jp2", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg2k_rgb", +) +``` + +
+ + +
+image_decoder_decode_jpeg_bgr + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="BGR", +) + +data, output = generate_test_data(".jpg", "BGR") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_bgr", +) +``` + +
+ + +
+image_decoder_decode_jpeg_grayscale + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="Grayscale", +) + +data, output = generate_test_data(".jpg", "Grayscale") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_grayscale", +) +``` + +
+ + +
+image_decoder_decode_jpeg_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".jpg", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_rgb", +) +``` + +
+ + +
+image_decoder_decode_png_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".png", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_png_rgb", +) +``` + +
+ + +
+image_decoder_decode_pnm_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".pnm", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_pnm_rgb", +) +``` + +
+ + +
+image_decoder_decode_tiff_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".tiff", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_tiff_rgb", +) +``` + +
+ + +
+image_decoder_decode_webp_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".webp", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_webp_rgb", +) +``` + +
+ + ### **InstanceNormalization** Carries out instance normalization as described in the paper diff --git a/docs/TestCoverage.md b/docs/TestCoverage.md index 3e2a2d8e77b..20b5d98888c 100644 --- a/docs/TestCoverage.md +++ b/docs/TestCoverage.md @@ -6,7 +6,7 @@ * [Overall Test Coverage](#overall-test-coverage) # Node Test Coverage ## Summary -Node tests have covered 178/191 (93.19%, 5 generators excluded) common operators. +Node tests have covered 179/192 (93.23%, 5 generators excluded) common operators. Node tests have covered 0/0 (N/A) experimental operators. @@ -8181,6 +8181,199 @@ expect( +### ImageDecoder +There are 9 test cases, listed as following: +
+image_decoder_decode_bmp_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".bmp", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_bmp_rgb", +) +``` + +
+
+image_decoder_decode_jpeg2k_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".jp2", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg2k_rgb", +) +``` + +
+
+image_decoder_decode_jpeg_bgr + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="BGR", +) + +data, output = generate_test_data(".jpg", "BGR") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_bgr", +) +``` + +
+
+image_decoder_decode_jpeg_grayscale + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="Grayscale", +) + +data, output = generate_test_data(".jpg", "Grayscale") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_grayscale", +) +``` + +
+
+image_decoder_decode_jpeg_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".jpg", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_rgb", +) +``` + +
+
+image_decoder_decode_png_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".png", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_png_rgb", +) +``` + +
+
+image_decoder_decode_pnm_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".pnm", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_pnm_rgb", +) +``` + +
+
+image_decoder_decode_tiff_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".tiff", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_tiff_rgb", +) +``` + +
+
+image_decoder_decode_webp_rgb + +```python +node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", +) + +data, output = generate_test_data(".webp", "RGB") +expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_webp_rgb", +) +``` + +
+ + ### InstanceNormalization There are 1 test cases, listed as following:
diff --git a/onnx/backend/test/case/node/image_decoder.py b/onnx/backend/test/case/node/image_decoder.py new file mode 100644 index 00000000000..69105dee62d --- /dev/null +++ b/onnx/backend/test/case/node/image_decoder.py @@ -0,0 +1,212 @@ +# Copyright (c) ONNX Project Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +import cv2 +import numpy as np + +import onnx +from onnx.backend.test.case.base import Base +from onnx.backend.test.case.node import expect + + +def generate_checkerboard(width, height, square_size): + # Create an empty RGB image + image = np.zeros((height, width, 3), dtype=np.uint8) + + # Calculate the number of squares in each dimension + num_squares_x = width // square_size + num_squares_y = height // square_size + + # Generate a random color for each square + colors = np.random.randint( + 0, 256, size=(num_squares_y, num_squares_x, 3), dtype=np.uint8 + ) + + # Iterate over each square + for i in range(num_squares_y): + for j in range(num_squares_x): + # Calculate the position of the current square + x = j * square_size + y = i * square_size + + # Get the color for the current square + color = colors[i, j] + + # Fill the square with the corresponding color + image[y : y + square_size, x : x + square_size, :] = color + + return image + + +def generate_test_data(extension, pixel_format="RGB", h=40, w=40, tile_sz=5): + data, output = None, None + np.random.seed(12345) + image = generate_checkerboard(h, w, tile_sz) + image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + _, encoded_image = cv2.imencode(extension, image_bgr) + data = np.frombuffer(encoded_image, dtype=np.uint8) + if pixel_format == "BGR": + output = cv2.imdecode(data, cv2.IMREAD_COLOR) + elif pixel_format == "RGB": + output_bgr = cv2.imdecode(data, cv2.IMREAD_COLOR) + output = cv2.cvtColor(output_bgr, cv2.COLOR_BGR2RGB) + elif pixel_format == "Grayscale": + output = cv2.imdecode(data, cv2.IMREAD_GRAYSCALE) + output = np.expand_dims(output, axis=2) # (H, W) to (H, W, 1) + return data, output + + +class ImageDecoder(Base): + @staticmethod + def export_image_decoder_decode_jpeg_rgb() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", + ) + + data, output = generate_test_data(".jpg", "RGB") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_rgb", + ) + + @staticmethod + def export_image_decoder_decode_jpeg_grayscale() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="Grayscale", + ) + + data, output = generate_test_data(".jpg", "Grayscale") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_grayscale", + ) + + @staticmethod + def export_image_decoder_decode_jpeg_bgr() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="BGR", + ) + + data, output = generate_test_data(".jpg", "BGR") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg_bgr", + ) + + @staticmethod + def export_image_decoder_decode_jpeg2k_rgb() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", + ) + + data, output = generate_test_data(".jp2", "RGB") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_jpeg2k_rgb", + ) + + @staticmethod + def export_image_decoder_decode_bmp_rgb() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", + ) + + data, output = generate_test_data(".bmp", "RGB") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_bmp_rgb", + ) + + @staticmethod + def export_image_decoder_decode_png_rgb() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", + ) + + data, output = generate_test_data(".png", "RGB") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_png_rgb", + ) + + @staticmethod + def export_image_decoder_decode_tiff_rgb() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", + ) + + data, output = generate_test_data(".tiff", "RGB") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_tiff_rgb", + ) + + @staticmethod + def export_image_decoder_decode_webp_rgb() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", + ) + + data, output = generate_test_data(".webp", "RGB") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_webp_rgb", + ) + + @staticmethod + def export_image_decoder_decode_pnm_rgb() -> None: + node = onnx.helper.make_node( + "ImageDecoder", + inputs=["data"], + outputs=["output"], + pixel_format="RGB", + ) + + data, output = generate_test_data(".pnm", "RGB") + expect( + node, + inputs=[data], + outputs=[output], + name="test_image_decoder_decode_pnm_rgb", + ) diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/model.onnx b/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/model.onnx new file mode 100644 index 00000000000..69eded0b8b4 Binary files /dev/null and b/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/model.onnx differ diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/test_data_set_0/input_0.pb new file mode 100644 index 00000000000..7b6e14abf86 Binary files /dev/null and b/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/test_data_set_0/input_0.pb differ diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/test_data_set_0/output_0.pb new file mode 100644 index 00000000000..6ed563ac91a --- /dev/null +++ b/onnx/backend/test/data/node/test_image_decoder_decode_bmp_rgb/test_data_set_0/output_0.pb @@ -0,0 +1,101 @@ +((BoutputJ%QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuuvlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_ZPZPZPZPZPvUvUvUvUvU'Ь'Ь'Ь'Ь'։^]\^_=<>?=spstr<;9=8Ў__\`[=??=@qrrrr;;:=;ԇ^^_^]==<>@qsrsq9::<:ԋ]]\^\>?D8>qsvpr=>;;9э``^`^=>;A@qqmst)+*,/றTWVVXuursrYZ[YY--,,,ݮYWWVVssrwuZ[ZXZ+++,+޲WTWXWtvust[YW]Y,+,-+యUYYTVwtrsuYZ]X]-+,-*ޯYWTYXrsuusY[YW[ \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/model.onnx b/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/model.onnx new file mode 100644 index 00000000000..d2dd7b9a208 Binary files /dev/null and b/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/model.onnx differ diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/test_data_set_0/input_0.pb new file mode 100644 index 00000000000..1773f14db39 Binary files /dev/null and b/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/test_data_set_0/input_0.pb differ diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/test_data_set_0/output_0.pb new file mode 100644 index 00000000000..bb2d5aa1168 Binary files /dev/null and b/onnx/backend/test/data/node/test_image_decoder_decode_jpeg_rgb/test_data_set_0/output_0.pb differ diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/model.onnx b/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/model.onnx new file mode 100644 index 00000000000..b33613c0076 Binary files /dev/null and b/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/model.onnx differ diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/test_data_set_0/input_0.pb new file mode 100644 index 00000000000..0a210d44d18 Binary files /dev/null and b/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/test_data_set_0/input_0.pb differ diff --git a/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/test_data_set_0/output_0.pb new file mode 100644 index 00000000000..6ed563ac91a --- /dev/null +++ b/onnx/backend/test/data/node/test_image_decoder_decode_png_rgb/test_data_set_0/output_0.pb @@ -0,0 +1,101 @@ +((BoutputJ%QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +QQQQQ-----PPPPPvvvvv!M!M!M!M!M/////- +- +- +- +- +^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuu^^^^^4~Q4~Q4~Q4~Q4~QeeeeemVmVmVmVmV"B6"B6"B6"B6"B6uuuuuvlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_vlvlvlvlvlqqqqq@v;@v;@v;@v;@v;66666zzzzziXiXiXiXiX[[[[[L_L_L_L_L_ZPZPZPZPZPvUvUvUvUvU'Ь'Ь'Ь'Ь' + +#include "onnx/defs/data_type_utils.h" +#include "onnx/defs/schema.h" +#include "onnx/defs/tensor_proto_util.h" + +namespace ONNX_NAMESPACE { + +static const char* ImageDecoder_ver20_doc = + R"DOC(Loads and decodes and image from a file. If it can't decode for any reason (e.g. corrupted encoded +stream, invalid format, it will return an empty matrix). +The following image formats are supported: +* BMP +* JPEG (note: Lossless JPEG support is optional) +* JPEG2000 +* TIFF +* PNG +* WebP +* Portable image format (PBM, PGM, PPM, PXM, PNM) +Decoded images follow a channel-last layout: (Height, Width, Channels). +**JPEG chroma upsampling method:** +When upsampling the chroma components by a factor of 2, the pixels are linearly interpolated so that the +centers of the output pixels are 1/4 and 3/4 of the way between input pixel centers. +When rounding, 0.5 is rounded down and up at alternative pixels locations to prevent bias towards +larger values (ordered dither pattern). +Considering adjacent input pixels A, B, and C, B is upsampled to pixels B0 and B1 so that +``` +B0 = round_half_down((1/4) * A + (3/4) * B) +B1 = round_half_up((3/4) * B + (1/4) * C) +``` +This method, is the default chroma upsampling method in the well-established libjpeg-turbo library, +also referred as "smooth" or "fancy" upsampling. +)DOC"; + +ONNX_OPERATOR_SET_SCHEMA( + ImageDecoder, + 20, + OpSchema() + .SetDoc(ImageDecoder_ver20_doc) + .Attr( + "pixel_format", + "Pixel format. Can be one of \"RGB\", \"BGR\", or \"Grayscale\".", + AttributeProto::STRING, + std::string("RGB")) + .Input(0, "encoded_stream", "Encoded stream", "T1", OpSchema::Single, true, 1, OpSchema::NonDifferentiable) + .Output(0, "image", "Decoded image", "T2", OpSchema::Single, true, 1, OpSchema::NonDifferentiable) + .TypeConstraint("T1", {"tensor(uint8)"}, "Constrain input types to 8-bit unsigned integer tensor.") + .TypeConstraint("T2", {"tensor(uint8)"}, "Constrain output types to 8-bit unsigned integer tensor.") + .TypeAndShapeInferenceFunction([](InferenceContext& ctx) { + if (hasInputShape(ctx, 0)) { + auto& input_shape = getInputShape(ctx, 0); + if (input_shape.dim_size() != 1) { + fail_shape_inference("Input tensor must be 1-dimensional"); + } + } + propagateElemTypeFromDtypeToOutput(ctx, TensorProto::UINT8, 0); + auto output_type = ctx.getOutputType(0); + auto* sh = output_type->mutable_tensor_type()->mutable_shape(); + sh->clear_dim(); + sh->add_dim(); + sh->add_dim(); + sh->add_dim(); + })); + +} // namespace ONNX_NAMESPACE diff --git a/onnx/defs/operator_sets.h b/onnx/defs/operator_sets.h index 7b500b01b32..c599f4c5ef4 100644 --- a/onnx/defs/operator_sets.h +++ b/onnx/defs/operator_sets.h @@ -1109,6 +1109,7 @@ class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 20, ConstantOfShape); class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 20, StringConcat); class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 20, RegexFullMatch); class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 20, StringSplit); +class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 20, ImageDecoder); // Iterate over schema from ai.onnx version 20 class OpSet_Onnx_ver20 { @@ -1121,6 +1122,7 @@ class OpSet_Onnx_ver20 { fn(GetOpSchema()); fn(GetOpSchema()); fn(GetOpSchema()); + fn(GetOpSchema()); } }; diff --git a/onnx/reference/ops/_op_list.py b/onnx/reference/ops/_op_list.py index 5caabeed7ce..c8a3fface44 100644 --- a/onnx/reference/ops/_op_list.py +++ b/onnx/reference/ops/_op_list.py @@ -114,6 +114,7 @@ from onnx.reference.ops.op_hardmax import Hardmax from onnx.reference.ops.op_identity import Identity from onnx.reference.ops.op_if import If +from onnx.reference.ops.op_image_decoder import ImageDecoder from onnx.reference.ops.op_instance_normalization import InstanceNormalization from onnx.reference.ops.op_isinf import IsInf from onnx.reference.ops.op_isnan import IsNaN diff --git a/onnx/reference/ops/op_image_decoder.py b/onnx/reference/ops/op_image_decoder.py new file mode 100644 index 00000000000..1b1090b478b --- /dev/null +++ b/onnx/reference/ops/op_image_decoder.py @@ -0,0 +1,29 @@ +# Copyright (c) ONNX Project Contributors + +# SPDX-License-Identifier: Apache-2.0 +# pylint: disable=C0123,C3001,R0912,R0913,R0914,R1730,W0221,W0613 + +import cv2 +import numpy as np + +from onnx.reference.op_run import OpRun + + +class ImageDecoder(OpRun): + def _run( # type: ignore + self, + encoded, + pixel_format="RGB", + ): + decoded = None + if pixel_format == "BGR": + decoded = cv2.imdecode(encoded, cv2.IMREAD_COLOR) + elif pixel_format == "RGB": + decoded = cv2.imdecode(encoded, cv2.IMREAD_COLOR) + decoded = cv2.cvtColor(decoded, cv2.COLOR_BGR2RGB) + elif pixel_format == "Grayscale": + decoded = cv2.imdecode(encoded, cv2.IMREAD_GRAYSCALE) + decoded = np.expand_dims(decoded, axis=2) # (H, W) to (H, W, 1) + else: + raise RuntimeError(f"pixel_format={pixel_format!r} is not supported.") + return (decoded,) diff --git a/onnx/test/automatic_upgrade_test.py b/onnx/test/automatic_upgrade_test.py index 8fa6dea19c5..94cb1146879 100644 --- a/onnx/test/automatic_upgrade_test.py +++ b/onnx/test/automatic_upgrade_test.py @@ -598,6 +598,16 @@ def test_If(self) -> None: attrs={"then_branch": then_graph, "else_branch": else_graph}, ) + def test_ImageDecoder(self) -> None: + self._test_op_upgrade( + "ImageDecoder", + 20, + [[None]], + [[None, None, 3]], + input_types=[TensorProto.UINT8], + output_types=[TensorProto.UINT8], + ) + def test_InstanceNormalization(self) -> None: self._test_op_upgrade( "InstanceNormalization", diff --git a/onnx/test/test_backend_onnxruntime.py b/onnx/test/test_backend_onnxruntime.py index 974e7e0c378..ca909797cb2 100644 --- a/onnx/test/test_backend_onnxruntime.py +++ b/onnx/test/test_backend_onnxruntime.py @@ -256,6 +256,7 @@ def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs): "|string_split" "|string_concat" "|gelu" + "|image_decoder" ")" ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 66aa8f93a9d..2da762ee3fd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,6 +11,6 @@ twine # Dependencies for linting. Versions match those in setup.py. lintrunner>=0.10.7 lintrunner-adapters>=0.8 -# Dependencies for the reference implementation. -google-re2 # Edit additional linter dependencies in requirements-lintrunner.txt +# Dependencies for the reference implementation. +-r requirements-reference.txt diff --git a/requirements-reference.txt b/requirements-reference.txt index 2347a47f5cb..d03b30f5747 100644 --- a/requirements-reference.txt +++ b/requirements-reference.txt @@ -1 +1,2 @@ google-re2 +opencv-python \ No newline at end of file