From e74ce7ce2942c5793e50f5ce199b5d14d06798e6 Mon Sep 17 00:00:00 2001 From: ddukbg Date: Sat, 2 Nov 2024 19:49:42 +0900 Subject: [PATCH] fix: Improve ZstdCompressor implementation Signed-off-by: ddukbg --- lib/fluent/plugin/out_s3.rb | 2 -- test/test_out_s3.rb | 25 +++++++++++++------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/fluent/plugin/out_s3.rb b/lib/fluent/plugin/out_s3.rb index 92d5aa6..8bf38ac 100644 --- a/lib/fluent/plugin/out_s3.rb +++ b/lib/fluent/plugin/out_s3.rb @@ -632,8 +632,6 @@ def compress(chunk, tmp) end class ZstdCompressor < Compressor - require 'zstd-ruby' - DEFAULT_LEVEL = 3 def initialize(level = nil) diff --git a/test/test_out_s3.rb b/test/test_out_s3.rb index 59f753f..b5cf1e0 100644 --- a/test/test_out_s3.rb +++ b/test/test_out_s3.rb @@ -467,28 +467,29 @@ def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholde def test_write_with_zstd setup_mocks(true) s3_local_file_path = "/tmp/s3-test.zst" - + expected_s3path = "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.zst" - + setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: expected_s3path) - + config = CONFIG_TIME_SLICE + "\nstore_as zstd\n" d = create_time_sliced_driver(config) - + time = event_time("2011-01-02 13:14:15 UTC") d.run(default_tag: "test") do d.feed(time, { "a" => 1 }) d.feed(time, { "a" => 2 }) end - - File.open(s3_local_file_path, 'rb') do |file| - compressed_data = file.read - uncompressed_data = Zstd.decompress(compressed_data) - expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + - %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n] - assert_equal expected_data, uncompressed_data + + data = File.binread(s3_local_file_path) + begin + uncompressed = Zstd.decompress(data) + expected = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n] + assert_equal expected, uncompressed + ensure + FileUtils.rm_f(s3_local_file_path) end - FileUtils.rm_f(s3_local_file_path) end class MockResponse