Skip to content

Commit

Permalink
fix: Improve ZstdCompressor implementation
Browse files Browse the repository at this point in the history
Signed-off-by: ddukbg <[email protected]>
  • Loading branch information
ddukbg committed Nov 2, 2024
1 parent 85e0f50 commit e74ce7c
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 14 deletions.
2 changes: 0 additions & 2 deletions lib/fluent/plugin/out_s3.rb
Original file line number Diff line number Diff line change
Expand Up @@ -632,8 +632,6 @@ def compress(chunk, tmp)
end

class ZstdCompressor < Compressor
require 'zstd-ruby'

DEFAULT_LEVEL = 3

def initialize(level = nil)
Expand Down
25 changes: 13 additions & 12 deletions test/test_out_s3.rb
Original file line number Diff line number Diff line change
Expand Up @@ -467,28 +467,29 @@ def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholde
def test_write_with_zstd
setup_mocks(true)
s3_local_file_path = "/tmp/s3-test.zst"

expected_s3path = "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.zst"

setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: expected_s3path)

config = CONFIG_TIME_SLICE + "\nstore_as zstd\n"
d = create_time_sliced_driver(config)

time = event_time("2011-01-02 13:14:15 UTC")
d.run(default_tag: "test") do
d.feed(time, { "a" => 1 })
d.feed(time, { "a" => 2 })
end

File.open(s3_local_file_path, 'rb') do |file|
compressed_data = file.read
uncompressed_data = Zstd.decompress(compressed_data)
expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
assert_equal expected_data, uncompressed_data

data = File.binread(s3_local_file_path)
begin
uncompressed = Zstd.decompress(data)
expected = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
assert_equal expected, uncompressed
ensure
FileUtils.rm_f(s3_local_file_path)
end
FileUtils.rm_f(s3_local_file_path)
end

class MockResponse
Expand Down

0 comments on commit e74ce7c

Please sign in to comment.