From f4857e132093eaa248a4b932d28656b82f20eec1 Mon Sep 17 00:00:00 2001 From: mzegla Date: Tue, 23 Jul 2024 12:19:28 +0200 Subject: [PATCH] gtest adjustment --- src/cpp/src/logit_processor.hpp | 13 +- src/cpp/src/sampler.hpp | 4 +- tests/cpp/logit_filtering.cpp | 214 ++++++++++++++++++-------------- 3 files changed, 134 insertions(+), 97 deletions(-) diff --git a/src/cpp/src/logit_processor.hpp b/src/cpp/src/logit_processor.hpp index 2131ec01cb..b845f9a638 100644 --- a/src/cpp/src/logit_processor.hpp +++ b/src/cpp/src/logit_processor.hpp @@ -19,7 +19,7 @@ struct Token { struct Logits { float * m_data = nullptr; size_t m_size; - // Late initialized + // Late initialized for top_p or top_k transforms std::vector m_vector; Logits(float* data, size_t size): m_data(data), m_size(size) {} @@ -29,8 +29,11 @@ struct Logits { OPENVINO_ASSERT(m_vector.size() == 0, "Logits vector already initialized"); m_vector.reserve(m_size); for (size_t i = 0; i < m_size; i++) - m_vector.emplace_back(m_data[i], i); - + m_vector.emplace_back(m_data[i], i); + } + + bool vector_initialized() const { + return m_vector.size() > 0; } void resize(size_t new_size) { @@ -56,7 +59,7 @@ class TopPFilter : public ILogitTransformer { TopPFilter(double top_p) : m_top_p(top_p) {} void apply(Logits& logits) override { - if (logits.m_vector.size() == 0) { + if (!logits.vector_initialized()) { // Initialize and sort vector logits.initialize_vector(); std::sort(logits.m_vector.begin(), logits.m_vector.end(), [](const Token& lhs, const Token& rhs) {return lhs.m_log_prob > rhs.m_log_prob; }); @@ -84,7 +87,7 @@ class TopKFilter : public ILogitTransformer { if (m_top_k >= logits.m_size) return; - if (logits.m_vector.size() == 0) { + if (!logits.vector_initialized()) { // Initialize and partially sort vector logits.initialize_vector(); std::partial_sort(logits.m_vector.begin(), logits.m_vector.begin() + m_top_k, logits.m_vector.end(), [](const Token& lhs, const Token& rhs) {return lhs.m_log_prob > rhs.m_log_prob; }); diff --git a/src/cpp/src/sampler.hpp b/src/cpp/src/sampler.hpp index 4c7ea52bed..65c17b4961 100644 --- a/src/cpp/src/sampler.hpp +++ b/src/cpp/src/sampler.hpp @@ -233,7 +233,7 @@ class Sampler { // If top_p or top_k was applied we use sorted vector, if not we go with original buffer. std::vector multinomial_weights; multinomial_weights.reserve(logits.m_size); - if (logits.m_vector.size() > 0) + if (logits.vector_initialized()) for (auto& logit: logits.m_vector) multinomial_weights.emplace_back(logit.m_log_prob); else multinomial_weights.assign(logits.m_data, logits.m_data + logits.m_size); @@ -243,7 +243,7 @@ class Sampler { std::vector out_tokens; for (size_t token_idx = 0; token_idx < num_tokens_per_sequence; ++token_idx) { size_t element_to_pick = dist(rng_engine); - if (logits.m_vector.size() > 0) + if (logits.vector_initialized()) out_tokens.push_back(logits.m_vector[element_to_pick]); else out_tokens.emplace_back(logits.m_data[element_to_pick], element_to_pick); diff --git a/tests/cpp/logit_filtering.cpp b/tests/cpp/logit_filtering.cpp index afedfe6685..26cf75b5c9 100644 --- a/tests/cpp/logit_filtering.cpp +++ b/tests/cpp/logit_filtering.cpp @@ -9,31 +9,32 @@ using namespace LogitTransformers; struct TemperatureTransformTestStruct { + static inline const size_t size = 3; + float temperature; - std::vector input; - std::vector expected_output; + float input[size]; + float expected_output[size]; }; using TemperatureTransformTest = testing::TestWithParam; TEST_P(TemperatureTransformTest, TransformResultEqualToReference) { auto test_struct = GetParam(); - auto logits = test_struct.input; + auto logits = Logits(test_struct.input, TemperatureTransformTestStruct::size); auto transform = TemperatureLogitTransform(test_struct.temperature); transform.apply(logits); - ASSERT_EQ(logits.size(), test_struct.expected_output.size()); - std::sort(logits.begin(), logits.end(), [](const Token& lhs, const Token& rhs) {return lhs.m_log_prob > rhs.m_log_prob; }); - for (size_t i = 0; i < logits.size(); i++) { - EXPECT_NEAR(logits[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); - EXPECT_EQ(logits[i].m_index, test_struct.expected_output[i].m_index); + ASSERT_FALSE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, TemperatureTransformTestStruct::size); // temperature transfrom should not change buffer size + for (size_t i = 0; i < logits.m_size; i++) { + EXPECT_NEAR(logits.m_data[i], test_struct.expected_output[i], 1e-6); } } const std::vector TEMPERATURE_TRANSFORM_TEST_CASES = { - {1.0f, { {1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, { {0.665241, 2}, {0.244728, 1}, {0.090031, 0} } }, - {2.0f, { {1.0f, 2}, {2.0f, 1}, {3.0f, 0} }, { {0.506480, 0}, {0.307195, 1}, {0.186323, 2} } }, - {1.0f, { {3.0f, 0}, {1.0f, 1}, {2.0f, 2} }, { {0.665241, 0}, {0.244728, 2}, {0.090031, 1} } }, + {1.0f, { 1.0f, 2.0f, 3.0f }, { 0.090031, 0.244728, 0.665241 } }, + {2.0f, { 3.0f, 2.0f, 1.0f }, { 0.506480, 0.307195, 0.186323 } }, + {1.0f, { 3.0f, 1.0f, 2.0f }, { 0.665241, 0.090031, 0.244728 } }, }; INSTANTIATE_TEST_SUITE_P(VariousInputs, @@ -43,8 +44,10 @@ INSTANTIATE_TEST_SUITE_P(VariousInputs, struct TopPTestStruct { + static inline const size_t size = 3; + float top_p; - std::vector input; + float input[size]; std::vector expected_output; }; @@ -52,21 +55,22 @@ using TopPFilteringTest = testing::TestWithParam; TEST_P(TopPFilteringTest, FilterResultEqualToReference) { auto test_struct = GetParam(); - auto logits = test_struct.input; + auto logits = Logits(test_struct.input, TopPTestStruct::size); auto transform = TopPFilter(test_struct.top_p); transform.apply(logits); - ASSERT_EQ(logits.size(), test_struct.expected_output.size()); - for (size_t i = 0; i < logits.size(); i++) { - EXPECT_NEAR(logits[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); - EXPECT_EQ(logits[i].m_index, test_struct.expected_output[i].m_index); + ASSERT_TRUE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, logits.m_vector.size()); + ASSERT_EQ(logits.m_size, test_struct.expected_output.size()); + for (size_t i = 0; i < logits.m_vector.size(); i++) { + EXPECT_NEAR(logits.m_vector[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); + EXPECT_EQ(logits.m_vector[i].m_index, test_struct.expected_output[i].m_index); } } const std::vector TOP_P_TRANSFORM_TEST_CASES = { - {0.2f, { {0.090031, 0}, {0.244728, 1}, {0.665241, 2} }, { {0.665241, 2} } }, - {0.9f, { {0.090031, 0}, {0.244728, 1}, {0.665241, 2} }, { {0.665241, 2}, {0.244728, 1} } }, - {1.0f, { {0.090031, 0}, {0.244728, 1}, {0.665241, 2} }, { {0.665241, 2}, {0.244728, 1}, {0.090031, 0} } }, + {0.2f, { 0.090031, 0.244728, 0.665241 }, { {0.665241, 2} } }, + {0.9f, { 0.090031, 0.244728, 0.665241 }, { {0.665241, 2}, {0.244728, 1} } }, }; INSTANTIATE_TEST_SUITE_P(VariousInputs, @@ -76,8 +80,10 @@ INSTANTIATE_TEST_SUITE_P(VariousInputs, struct TopKTestStruct { + static inline const size_t size = 3; + size_t top_k; - std::vector input; + float input[size]; std::vector expected_output; }; @@ -85,45 +91,62 @@ using TopKFilteringTest = testing::TestWithParam; TEST_P(TopKFilteringTest, FilterResultEqualToReference) { auto test_struct = GetParam(); - auto logits = test_struct.input; + auto logits = Logits(test_struct.input, TopKTestStruct::size); auto transform = TopKFilter(test_struct.top_k); transform.apply(logits); - ASSERT_EQ(logits.size(), test_struct.expected_output.size()); - for (size_t i = 0; i < logits.size(); i++) { - EXPECT_NEAR(logits[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); - EXPECT_EQ(logits[i].m_index, test_struct.expected_output[i].m_index); + ASSERT_TRUE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, logits.m_vector.size()); + ASSERT_EQ(logits.m_size, test_struct.expected_output.size()); + for (size_t i = 0; i < logits.m_vector.size(); i++) { + EXPECT_NEAR(logits.m_vector[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); + EXPECT_EQ(logits.m_vector[i].m_index, test_struct.expected_output[i].m_index); } } const std::vector TOP_K_TRANSFORM_TEST_CASES = { - {1, { {0.090031, 0}, {0.244728, 1}, {0.665241, 2} }, { {0.665241, 2} } }, - {2, { {0.090031, 0}, {0.244728, 1}, {0.665241, 2} }, { {0.665241, 2}, {0.244728, 1} } }, - {5, { {0.090031, 0}, {0.244728, 1}, {0.665241, 2} }, { {0.665241, 2}, {0.244728, 1}, {0.090031, 0} } }, + {1, { 0.090031, 0.244728, 0.665241 }, { {0.665241, 2} } }, + {2, { 0.090031, 0.244728, 0.665241 }, { {0.665241, 2}, {0.244728, 1} } }, }; INSTANTIATE_TEST_SUITE_P(VariousInputs, TopKFilteringTest, testing::ValuesIn(TOP_K_TRANSFORM_TEST_CASES)); +TEST(TopKFilteringTest, FilterNotAppliedTopKGreaterThanInputSize) { + float input[]{0.090031, 0.244728, 0.665241}; + float expected_output[]{0.090031, 0.244728, 0.665241}; // no change expected + size_t top_k = 5; + auto logits = Logits(input, 3); + auto transform = TopKFilter(top_k); + transform.apply(logits); + ASSERT_FALSE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, 3); + for (size_t i = 0; i < logits.m_size; i++) { + EXPECT_EQ(logits.m_data[i], expected_output[i]); + } +} + struct RepetitionPenaltyTransformTestStruct { + static inline const size_t size = 3; + float penalty; - std::vector input; + float input[size]; TokenIds input_ids; - std::vector expected_output; + float expected_output[size]; }; using RepetitionPenaltyTransformTest = testing::TestWithParam; TEST_P(RepetitionPenaltyTransformTest, TransformResultEqualToReference) { auto test_struct = GetParam(); - auto logits = test_struct.input; + auto logits = Logits(test_struct.input, RepetitionPenaltyTransformTestStruct::size); auto transform = RepetitionPenaltyTransform(test_struct.penalty); transform.apply(logits, test_struct.input_ids); - ASSERT_EQ(logits.size(), test_struct.expected_output.size()); - for (size_t i = 0; i < logits.size(); i++) { - EXPECT_NEAR(logits[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); - EXPECT_EQ(logits[i].m_index, test_struct.expected_output[i].m_index); + ASSERT_FALSE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, RepetitionPenaltyTransformTestStruct::size); // penalty transfrom should not change buffer size + for (size_t i = 0; i < logits.m_size; i++) { + EXPECT_NEAR(logits.m_data[i], test_struct.expected_output[i], 1e-6); } } @@ -131,21 +154,21 @@ TEST_P(RepetitionPenaltyTransformTest, TransformResultEqualToReference) { const std::vector REPETITION_PENALTY_TRANSFORM_TEST_CASES = { { // basic case, indices are applied, order is left as-is 1.2f, - { {1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { 1.0f, 2.0f, 3.0f }, { 2, 0 }, - { {0.8333333f, 0}, {2.0f, 1}, {2.5f, 2} } + { 0.8333333f, 2.0f, 2.5f } }, { // negative scores case 2.0f, - { {-1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { -1.0f, 2.0f, 3.0f }, { 0, 1 }, - { {-2.0f, 0}, {1.0f, 1}, {3.0f, 2} } + { -2.0f, 1.0f, 3.0f } }, { // repeated tokens in prompt, check that the penalty is only applied once 0.5f, - { {-1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { -1.0f, 2.0f, 3.0f }, { 1, 1 }, - { {-1.0f, 0}, {4.0f, 1}, {3.0f, 2} } + { -1.0f, 4.0f, 3.0f } }, }; @@ -155,30 +178,34 @@ INSTANTIATE_TEST_SUITE_P(VariousInputs, TEST(RepetitionPenaltyTransformInitializationTest, ThrowsForInvalidInputIds) { auto transform = RepetitionPenaltyTransform(1.5); - std::vector input {{43.0f, 0}}; - EXPECT_THROW(transform.apply(input, {1337}), ov::Exception); - input = {{18.0f, 0}}; - EXPECT_THROW(transform.apply(input, {0, -1}), ov::Exception); + float input[]{43.0f}; + Logits logits(input, 1); + EXPECT_THROW(transform.apply(logits, {1337}), ov::Exception); + input[0] = {18.0f}; + EXPECT_THROW(transform.apply(logits, {0, -1}), ov::Exception); } + struct FrequencyPenaltyTransformTestStruct { + static inline const size_t size = 3; + float penalty; - std::vector input; + float input[size]; TokenIds input_ids; - std::vector expected_output; + float expected_output[size]; }; using FrequencyPenaltyTransformTest = testing::TestWithParam; TEST_P(FrequencyPenaltyTransformTest, TransformResultEqualToReference) { auto test_struct = GetParam(); - auto logits = test_struct.input; + auto logits = Logits(test_struct.input, FrequencyPenaltyTransformTestStruct::size); auto transform = FrequencyPenaltyTransform(test_struct.penalty); transform.apply(logits, test_struct.input_ids); - ASSERT_EQ(logits.size(), test_struct.expected_output.size()); - for (size_t i = 0; i < logits.size(); i++) { - EXPECT_NEAR(logits[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); - EXPECT_EQ(logits[i].m_index, test_struct.expected_output[i].m_index); + ASSERT_FALSE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, FrequencyPenaltyTransformTestStruct::size); // penalty transfrom should not change buffer size + for (size_t i = 0; i < logits.m_size; i++) { + EXPECT_NEAR(logits.m_data[i], test_struct.expected_output[i], 1e-6); } }; @@ -186,21 +213,21 @@ TEST_P(FrequencyPenaltyTransformTest, TransformResultEqualToReference) { const std::vector FREQUENCY_PENALTY_TRANSFORM_TEST_CASES = { { // basic case, indices are applied, order is left as-is 0.5f, - { {-1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { -1.0f, 2.0f, 3.0f }, { 1, 0 }, - { {-0.5f, 0}, {1.5f, 1}, {3.0f, 2} } + { -0.5f, 1.5f, 3.0f } }, { // negative scores case -0.6f, - { {-1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { -1.0f, 2.0f, 3.0f }, { 0, 1, 1 }, - { {-1.6f, 0}, {3.2f, 1}, {3.0f, 2} } + { -1.6f, 3.2f, 3.0f } }, { // repeated tokens in prompt, check that the penalty is only applied once 0.2f, - { {1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { 1.0f, 2.0f, 3.0f }, { 2, 0, 2 }, - { {0.8f, 0}, {2.0f, 1}, {2.6f, 2} } + { 0.8f, 2.0f, 2.6f } }, }; @@ -210,31 +237,34 @@ INSTANTIATE_TEST_SUITE_P(VariousInputs, TEST(FrequencyPenaltyTransformInitializationTest, ThrowsForInvalidInputIds) { auto transform = FrequencyPenaltyTransform(1.5); - std::vector input {{43.0f, 0}}; - EXPECT_THROW(transform.apply(input, {1337}), ov::Exception); - input = {{18.0f, 0}}; - EXPECT_THROW(transform.apply(input, {0, -1}), ov::Exception); + float input[]{43.0f}; + Logits logits(input, 1); + EXPECT_THROW(transform.apply(logits, {1337}), ov::Exception); + input[0] = {18.0f}; + EXPECT_THROW(transform.apply(logits, {0, -1}), ov::Exception); } struct PresencePenaltyTransformTestStruct { + static inline const size_t size = 3; + float penalty; - std::vector input; + float input[size]; TokenIds input_ids; - std::vector expected_output; + float expected_output[size]; }; using PresencePenaltyTransformTest = testing::TestWithParam; TEST_P(PresencePenaltyTransformTest, TransformResultEqualToReference) { auto test_struct = GetParam(); - auto logits = test_struct.input; + auto logits = Logits(test_struct.input, PresencePenaltyTransformTestStruct::size); auto transform = PresencePenaltyTransform(test_struct.penalty); transform.apply(logits, test_struct.input_ids); - ASSERT_EQ(logits.size(), test_struct.expected_output.size()); - for (size_t i = 0; i < logits.size(); i++) { - EXPECT_NEAR(logits[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); - EXPECT_EQ(logits[i].m_index, test_struct.expected_output[i].m_index); + ASSERT_FALSE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, PresencePenaltyTransformTestStruct::size); // penalty transfrom should not change buffer size + for (size_t i = 0; i < logits.m_size; i++) { + EXPECT_NEAR(logits.m_data[i], test_struct.expected_output[i], 1e-6); } }; @@ -242,21 +272,21 @@ TEST_P(PresencePenaltyTransformTest, TransformResultEqualToReference) { const std::vector PRESENCE_PENALTY_TRANSFORM_TEST_CASES = { { // basic case, indices are applied, order is left as-is 0.5f, - { {-1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { -1.0f, 2.0f, 3.0f }, { 1, 0 }, - { {-0.5f, 0}, {1.5f, 1}, {3.0f, 2} } + { -0.5f, 1.5f, 3.0f } }, { // negative scores case -0.6f, - { {-1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { -1.0f, 2.0f, 3.0f }, { 0, 1, 1 }, - { {-1.6f, 0}, {2.6f, 1}, {3.0f, 2} } + { -1.6f, 2.6f, 3.0f } }, { // repeated tokens in prompt, check that the penalty is only applied once 0.2f, - { {1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, + { 1.0f, 2.0f, 3.0f }, { 2, 0, 2 }, - { {0.8f, 0}, {2.0f, 1}, {2.8f, 2} } + { 0.8f, 2.0f, 2.8f } }, }; @@ -266,29 +296,32 @@ INSTANTIATE_TEST_SUITE_P(VariousInputs, TEST(PresencePenaltyTransformInitializationTest, ThrowsForInvalidInputIds) { auto transform = PresencePenaltyTransform(1.5); - std::vector input {{43.0f, 0}}; - EXPECT_THROW(transform.apply(input, {1337}), ov::Exception); - input = {{18.0f, 0}}; - EXPECT_THROW(transform.apply(input, {0, -1}), ov::Exception); + float input[]{43.0f}; + Logits logits(input, 1); + EXPECT_THROW(transform.apply(logits, {1337}), ov::Exception); + input[0] = {18.0f}; + EXPECT_THROW(transform.apply(logits, {0, -1}), ov::Exception); } struct EOSPenaltyTransformTestStruct { + static inline const size_t size = 3; + size_t eos_token_id; - std::vector input; - std::vector expected_output; + float input[size]; + float expected_output[size]; }; using EOSPenaltyTransformTest = testing::TestWithParam; TEST_P(EOSPenaltyTransformTest, TransformResultEqualToReference) { auto test_struct = GetParam(); - auto logits = test_struct.input; + auto logits = Logits(test_struct.input, EOSPenaltyTransformTestStruct::size); auto transform = EOSPenaltyTransform(test_struct.eos_token_id, std::numeric_limits::max()); transform.apply(logits); - ASSERT_EQ(logits.size(), test_struct.expected_output.size()); - for (size_t i = 0; i < logits.size(); i++) { - EXPECT_NEAR(logits[i].m_log_prob, test_struct.expected_output[i].m_log_prob, 1e-6); - EXPECT_EQ(logits[i].m_index, test_struct.expected_output[i].m_index); + ASSERT_FALSE(logits.vector_initialized()); + ASSERT_EQ(logits.m_size, EOSPenaltyTransformTestStruct::size); // penalty transfrom should not change buffer size + for (size_t i = 0; i < logits.m_size; i++) { + EXPECT_NEAR(logits.m_data[i], test_struct.expected_output[i], 1e-6); } } @@ -296,11 +329,12 @@ TEST_P(EOSPenaltyTransformTest, TransformResultEqualToReference) { const std::vector EOS_PENALTY_TRANSFORM_TEST_CASES = { { // basic case, indices are applied, order is left as-is 1, - { {1.0f, 0}, {2.0f, 1}, {3.0f, 2} }, - { {1.0f, 0}, {0.0f, 1}, {3.0f, 2} }, + { 1.0f, 2.0f, 3.0f }, + { 1.0f, 0.0f, 3.0f }, }, }; INSTANTIATE_TEST_SUITE_P(VariousInputs, EOSPenaltyTransformTest, - testing::ValuesIn(EOS_PENALTY_TRANSFORM_TEST_CASES)); \ No newline at end of file + testing::ValuesIn(EOS_PENALTY_TRANSFORM_TEST_CASES)); +