From 26be9516cd81d5315ee31c48d2438018e0eab879 Mon Sep 17 00:00:00 2001 From: Taku Kudo Date: Mon, 8 Jun 2020 16:21:32 +0900 Subject: [PATCH] upgrade protobuf --- src/CMakeLists.txt | 4 +- src/builtin_pb/sentencepiece.pb.cc | 933 ++-- src/builtin_pb/sentencepiece.pb.h | 1011 ++-- src/builtin_pb/sentencepiece_model.pb.cc | 3515 ++++++------ src/builtin_pb/sentencepiece_model.pb.h | 4857 ++++++++++------- src/trainer_interface.cc | 13 +- third_party/protobuf-lite/arena.cc | 101 +- third_party/protobuf-lite/bytestream.cc | 4 +- third_party/protobuf-lite/coded_stream.cc | 540 +- third_party/protobuf-lite/common.cc | 100 +- third_party/protobuf-lite/extension_set.cc | 1347 +++-- ...ed_stream_inl.h => generated_enum_util.cc} | 89 +- .../generated_message_table_driven_lite.cc | 55 +- .../protobuf-lite/generated_message_util.cc | 169 +- .../protobuf-lite/google/protobuf/arena.h | 271 +- .../google/protobuf/arena_impl.h | 132 +- .../google/protobuf/arenastring.h | 116 +- .../google/protobuf/extension_set.h | 772 +-- .../google/protobuf/extension_set_inl.h | 274 + .../google/protobuf/generated_enum_util.h | 41 +- .../protobuf/generated_message_reflection.h | 317 ++ .../protobuf/generated_message_table_driven.h | 161 +- .../generated_message_table_driven_lite.h | 378 +- .../google/protobuf/generated_message_util.h | 297 +- .../protobuf-lite/google/protobuf/has_bits.h | 29 +- .../google/protobuf/implicit_weak_message.h | 137 +- .../google/protobuf/inlined_string_field.h | 205 +- .../google/protobuf/io/coded_stream.h | 990 ++-- .../google/protobuf/{stubs => io}/io_win32.h | 78 +- .../google/protobuf/io/zero_copy_stream.h | 17 +- .../protobuf/io/zero_copy_stream_impl_lite.h | 131 +- .../protobuf-lite/google/protobuf/map.h | 565 +- .../google/protobuf/map_entry_lite.h | 455 +- .../google/protobuf/map_field_lite.h | 81 +- .../google/protobuf/map_type_handler.h | 654 ++- .../google/protobuf/message_lite.h | 305 +- .../google/protobuf/metadata_lite.h | 186 +- .../google/protobuf/parse_context.h | 809 +++ .../protobuf/port.h} | 18 +- .../google/protobuf/port_def.inc | 486 ++ .../google/protobuf/port_undef.inc | 95 + .../google/protobuf/repeated_field.h | 1150 ++-- .../google/protobuf/stubs/bytestream.h | 49 +- .../google/protobuf/stubs/callback.h | 76 +- .../google/protobuf/stubs/casts.h | 17 +- .../google/protobuf/stubs/common.h | 88 +- .../google/protobuf/stubs/fastmem.h | 6 +- .../google/protobuf/stubs/hash.h | 324 +- .../google/protobuf/stubs/int128.h | 16 +- .../google/protobuf/stubs/logging.h | 32 +- .../google/protobuf/stubs/macros.h | 48 - .../google/protobuf/stubs/map_util.h | 40 +- .../google/protobuf/stubs/mathlimits.h | 303 - .../google/protobuf/stubs/mutex.h | 76 +- .../google/protobuf/stubs/platform_macros.h | 8 +- .../google/protobuf/stubs/port.h | 542 -- .../google/protobuf/stubs/status.h | 17 +- .../google/protobuf/stubs/statusor.h | 29 +- .../google/protobuf/stubs/stl_util.h | 52 +- .../google/protobuf/stubs/stringpiece.h | 55 +- .../google/protobuf/stubs/stringprintf.h | 19 +- .../google/protobuf/stubs/strutil.h | 322 +- .../google/protobuf/stubs/time.h | 17 +- .../google/protobuf/wire_format_lite.h | 1683 ++++-- .../google/protobuf/wire_format_lite_inl.h | 996 ---- .../protobuf-lite/implicit_weak_message.cc | 20 +- third_party/protobuf-lite/int128.cc | 71 +- third_party/protobuf-lite/io_win32.cc | 90 +- third_party/protobuf-lite/message_lite.cc | 522 +- third_party/protobuf-lite/parse_context.cc | 579 ++ third_party/protobuf-lite/repeated_field.cc | 38 +- third_party/protobuf-lite/statusor.cc | 2 + third_party/protobuf-lite/stringpiece.cc | 4 +- third_party/protobuf-lite/stringprintf.cc | 4 +- .../protobuf-lite/structurally_valid.cc | 9 +- third_party/protobuf-lite/strutil.cc | 330 +- third_party/protobuf-lite/time.cc | 40 +- third_party/protobuf-lite/wire_format_lite.cc | 299 +- .../protobuf-lite/zero_copy_stream_impl.cc | 375 ++ .../zero_copy_stream_impl_lite.cc | 163 +- 80 files changed, 17253 insertions(+), 11996 deletions(-) rename third_party/protobuf-lite/{google/protobuf/io/coded_stream_inl.h => generated_enum_util.cc} (50%) create mode 100644 third_party/protobuf-lite/google/protobuf/extension_set_inl.h create mode 100644 third_party/protobuf-lite/google/protobuf/generated_message_reflection.h rename third_party/protobuf-lite/google/protobuf/{stubs => io}/io_win32.h (58%) create mode 100644 third_party/protobuf-lite/google/protobuf/parse_context.h rename third_party/protobuf-lite/{arenastring.cc => google/protobuf/port.h} (81%) create mode 100644 third_party/protobuf-lite/google/protobuf/port_def.inc create mode 100644 third_party/protobuf-lite/google/protobuf/port_undef.inc delete mode 100644 third_party/protobuf-lite/google/protobuf/stubs/mathlimits.h delete mode 100644 third_party/protobuf-lite/google/protobuf/stubs/port.h delete mode 100644 third_party/protobuf-lite/google/protobuf/wire_format_lite_inl.h create mode 100644 third_party/protobuf-lite/parse_context.cc create mode 100644 third_party/protobuf-lite/zero_copy_stream_impl.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 07316a19..511b2ec1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,17 +20,18 @@ if (SPM_USE_BUILTIN_PROTOBUF) set(PROTOBUF_LITE_LIBRARY "") set(PROTOBUF_LITE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/arena.cc - ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/arenastring.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/bytestream.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/coded_stream.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/common.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/extension_set.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/generated_enum_util.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/generated_message_table_driven_lite.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/generated_message_util.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/implicit_weak_message.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/int128.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/io_win32.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/message_lite.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/parse_context.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/repeated_field.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/status.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/statusor.cc @@ -41,6 +42,7 @@ if (SPM_USE_BUILTIN_PROTOBUF) ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/time.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/wire_format_lite.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/zero_copy_stream.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/zero_copy_stream_impl.cc ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/zero_copy_stream_impl_lite.cc) if (MSVC) add_definitions("/DHAVE_PTHREAD /wd4018 /wd4514") diff --git a/src/builtin_pb/sentencepiece.pb.cc b/src/builtin_pb/sentencepiece.pb.cc index 0da509f7..695f18bf 100644 --- a/src/builtin_pb/sentencepiece.pb.cc +++ b/src/builtin_pb/sentencepiece.pb.cc @@ -5,124 +5,119 @@ #include -#include -#include #include -#include +#include +#include #include -// This is a temporary google only hack -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS -#include "third_party/protobuf/version.h" -#endif // @@protoc_insertion_point(includes) - -namespace protobuf_sentencepiece_2eproto { -extern PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_SentencePieceText_SentencePiece; -extern PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_2eproto ::google::protobuf::internal::SCCInfo<1> scc_info_SentencePieceText; -} // namespace protobuf_sentencepiece_2eproto +#include +extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SentencePieceText_sentencepiece_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SentencePieceText_SentencePiece_sentencepiece_2eproto; namespace sentencepiece { class SentencePieceText_SentencePieceDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _SentencePieceText_SentencePiece_default_instance_; class SentencePieceTextDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _SentencePieceText_default_instance_; class NBestSentencePieceTextDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _NBestSentencePieceText_default_instance_; } // namespace sentencepiece -namespace protobuf_sentencepiece_2eproto { -static void InitDefaultsSentencePieceText_SentencePiece() { +static void InitDefaultsscc_info_NBestSentencePieceText_sentencepiece_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { - void* ptr = &::sentencepiece::_SentencePieceText_SentencePiece_default_instance_; - new (ptr) ::sentencepiece::SentencePieceText_SentencePiece(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + void* ptr = &::sentencepiece::_NBestSentencePieceText_default_instance_; + new (ptr) ::sentencepiece::NBestSentencePieceText(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::sentencepiece::SentencePieceText_SentencePiece::InitAsDefaultInstance(); + ::sentencepiece::NBestSentencePieceText::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<0> scc_info_SentencePieceText_SentencePiece = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsSentencePieceText_SentencePiece}, {}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_NBestSentencePieceText_sentencepiece_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, 0, InitDefaultsscc_info_NBestSentencePieceText_sentencepiece_2eproto}, { + &scc_info_SentencePieceText_sentencepiece_2eproto.base,}}; -static void InitDefaultsSentencePieceText() { +static void InitDefaultsscc_info_SentencePieceText_sentencepiece_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { void* ptr = &::sentencepiece::_SentencePieceText_default_instance_; new (ptr) ::sentencepiece::SentencePieceText(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } ::sentencepiece::SentencePieceText::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<1> scc_info_SentencePieceText = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsSentencePieceText}, { - &protobuf_sentencepiece_2eproto::scc_info_SentencePieceText_SentencePiece.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SentencePieceText_sentencepiece_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, 0, InitDefaultsscc_info_SentencePieceText_sentencepiece_2eproto}, { + &scc_info_SentencePieceText_SentencePiece_sentencepiece_2eproto.base,}}; -static void InitDefaultsNBestSentencePieceText() { +static void InitDefaultsscc_info_SentencePieceText_SentencePiece_sentencepiece_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { - void* ptr = &::sentencepiece::_NBestSentencePieceText_default_instance_; - new (ptr) ::sentencepiece::NBestSentencePieceText(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + void* ptr = &::sentencepiece::_SentencePieceText_SentencePiece_default_instance_; + new (ptr) ::sentencepiece::SentencePieceText_SentencePiece(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::sentencepiece::NBestSentencePieceText::InitAsDefaultInstance(); + ::sentencepiece::SentencePieceText_SentencePiece::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<1> scc_info_NBestSentencePieceText = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsNBestSentencePieceText}, { - &protobuf_sentencepiece_2eproto::scc_info_SentencePieceText.base,}}; - -void InitDefaults() { - ::google::protobuf::internal::InitSCC(&scc_info_SentencePieceText_SentencePiece.base); - ::google::protobuf::internal::InitSCC(&scc_info_SentencePieceText.base); - ::google::protobuf::internal::InitSCC(&scc_info_NBestSentencePieceText.base); -} +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SentencePieceText_SentencePiece_sentencepiece_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_SentencePieceText_SentencePiece_sentencepiece_2eproto}, {}}; -} // namespace protobuf_sentencepiece_2eproto namespace sentencepiece { // =================================================================== void SentencePieceText_SentencePiece::InitAsDefaultInstance() { } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int SentencePieceText_SentencePiece::kPieceFieldNumber; -const int SentencePieceText_SentencePiece::kIdFieldNumber; -const int SentencePieceText_SentencePiece::kSurfaceFieldNumber; -const int SentencePieceText_SentencePiece::kBeginFieldNumber; -const int SentencePieceText_SentencePiece::kEndFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -SentencePieceText_SentencePiece::SentencePieceText_SentencePiece() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_2eproto::scc_info_SentencePieceText_SentencePiece.base); +class SentencePieceText_SentencePiece::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static void set_has_piece(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static void set_has_id(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } + static void set_has_surface(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } + static void set_has_begin(HasBits* has_bits) { + (*has_bits)[0] |= 8u; + } + static void set_has_end(HasBits* has_bits) { + (*has_bits)[0] |= 16u; + } +}; + +SentencePieceText_SentencePiece::SentencePieceText_SentencePiece(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + _extensions_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.SentencePieceText.SentencePiece) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.SentencePieceText.SentencePiece) } SentencePieceText_SentencePiece::SentencePieceText_SentencePiece(const SentencePieceText_SentencePiece& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); _extensions_.MergeFrom(from._extensions_); - piece_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_piece()) { - piece_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.piece_); + piece_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_piece()) { + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_piece(), + GetArena()); } - surface_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_surface()) { - surface_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.surface_); + surface_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_surface()) { + surface_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_surface(), + GetArena()); } ::memcpy(&id_, &from.id_, static_cast(reinterpret_cast(&end_) - @@ -131,8 +126,9 @@ SentencePieceText_SentencePiece::SentencePieceText_SentencePiece(const SentenceP } void SentencePieceText_SentencePiece::SharedCtor() { - piece_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - surface_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SentencePieceText_SentencePiece_sentencepiece_2eproto.base); + piece_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + surface_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&id_, 0, static_cast( reinterpret_cast(&end_) - reinterpret_cast(&id_)) + sizeof(end_)); @@ -141,196 +137,180 @@ void SentencePieceText_SentencePiece::SharedCtor() { SentencePieceText_SentencePiece::~SentencePieceText_SentencePiece() { // @@protoc_insertion_point(destructor:sentencepiece.SentencePieceText.SentencePiece) SharedDtor(); + _internal_metadata_.Delete(); } void SentencePieceText_SentencePiece::SharedDtor() { - piece_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - surface_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + GOOGLE_DCHECK(GetArena() == nullptr); + piece_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + surface_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } +void SentencePieceText_SentencePiece::ArenaDtor(void* object) { + SentencePieceText_SentencePiece* _this = reinterpret_cast< SentencePieceText_SentencePiece* >(object); + (void)_this; +} +void SentencePieceText_SentencePiece::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void SentencePieceText_SentencePiece::SetCachedSize(int size) const { _cached_size_.Set(size); } const SentencePieceText_SentencePiece& SentencePieceText_SentencePiece::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_2eproto::scc_info_SentencePieceText_SentencePiece.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_SentencePieceText_SentencePiece_sentencepiece_2eproto.base); return *internal_default_instance(); } void SentencePieceText_SentencePiece::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.SentencePieceText.SentencePiece) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _extensions_.Clear(); cached_has_bits = _has_bits_[0]; - if (cached_has_bits & 3u) { + if (cached_has_bits & 0x00000003u) { if (cached_has_bits & 0x00000001u) { - piece_.ClearNonDefaultToEmptyNoArena(); + piece_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000002u) { - surface_.ClearNonDefaultToEmptyNoArena(); + surface_.ClearNonDefaultToEmpty(); } } - if (cached_has_bits & 28u) { + if (cached_has_bits & 0x0000001cu) { ::memset(&id_, 0, static_cast( reinterpret_cast(&end_) - reinterpret_cast(&id_)) + sizeof(end_)); } _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool SentencePieceText_SentencePiece::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.SentencePieceText.SentencePiece) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + _internal_metadata_.Clear(); +} + +const char* SentencePieceText_SentencePiece::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { // optional string piece = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_piece())); - } else { - goto handle_unusual; - } - break; - } - + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + auto str = _internal_mutable_piece(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional uint32 id = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(16u /* 16 & 0xFF */)) { - set_has_id(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::uint32, ::google::protobuf::internal::WireFormatLite::TYPE_UINT32>( - input, &id_))); - } else { - goto handle_unusual; - } - break; - } - + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + _Internal::set_has_id(&has_bits); + id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string surface = 3; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_surface())); - } else { - goto handle_unusual; - } - break; - } - + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { + auto str = _internal_mutable_surface(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional uint32 begin = 4; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(32u /* 32 & 0xFF */)) { - set_has_begin(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::uint32, ::google::protobuf::internal::WireFormatLite::TYPE_UINT32>( - input, &begin_))); - } else { - goto handle_unusual; - } - break; - } - + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { + _Internal::set_has_begin(&has_bits); + begin_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional uint32 end = 5; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(40u /* 40 & 0xFF */)) { - set_has_end(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::uint32, ::google::protobuf::internal::WireFormatLite::TYPE_UINT32>( - input, &end_))); - } else { - goto handle_unusual; - } - break; - } - + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) { + _Internal::set_has_end(&has_bits); + end_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - if ((1600u <= tag)) { - DO_(_extensions_.ParseField(tag, input, - internal_default_instance(), - &unknown_fields_stream)); - continue; - } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + if ((1600u <= tag)) { + ptr = _extensions_.ParseField(tag, ptr, + internal_default_instance(), &_internal_metadata_, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.SentencePieceText.SentencePiece) - return true; + _has_bits_.Or(has_bits); + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.SentencePieceText.SentencePiece) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void SentencePieceText_SentencePiece::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.SentencePieceText.SentencePiece) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* SentencePieceText_SentencePiece::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.SentencePieceText.SentencePiece) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = _has_bits_[0]; // optional string piece = 1; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->piece(), output); + target = stream->WriteStringMaybeAliased( + 1, this->_internal_piece(), target); } // optional uint32 id = 2; if (cached_has_bits & 0x00000004u) { - ::google::protobuf::internal::WireFormatLite::WriteUInt32(2, this->id(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt32ToArray(2, this->_internal_id(), target); } // optional string surface = 3; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 3, this->surface(), output); + target = stream->WriteStringMaybeAliased( + 3, this->_internal_surface(), target); } // optional uint32 begin = 4; if (cached_has_bits & 0x00000008u) { - ::google::protobuf::internal::WireFormatLite::WriteUInt32(4, this->begin(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt32ToArray(4, this->_internal_begin(), target); } // optional uint32 end = 5; if (cached_has_bits & 0x00000010u) { - ::google::protobuf::internal::WireFormatLite::WriteUInt32(5, this->end(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt32ToArray(5, this->_internal_end(), target); } // Extension range [200, 536870912) - _extensions_.SerializeWithCachedSizes( - 200, 536870912, output); + target = _extensions_._InternalSerialize( + 200, 536870912, target, stream); - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.SentencePieceText.SentencePiece) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.SentencePieceText.SentencePiece) + return target; } size_t SentencePieceText_SentencePiece::ByteSizeLong() const { @@ -339,72 +319,77 @@ size_t SentencePieceText_SentencePiece::ByteSizeLong() const { total_size += _extensions_.ByteSize(); - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; - if (_has_bits_[0 / 32] & 31u) { + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x0000001fu) { // optional string piece = 1; - if (has_piece()) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->piece()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_piece()); } // optional string surface = 3; - if (has_surface()) { + if (cached_has_bits & 0x00000002u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->surface()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_surface()); } // optional uint32 id = 2; - if (has_id()) { + if (cached_has_bits & 0x00000004u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::UInt32Size( - this->id()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::UInt32Size( + this->_internal_id()); } // optional uint32 begin = 4; - if (has_begin()) { + if (cached_has_bits & 0x00000008u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::UInt32Size( - this->begin()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::UInt32Size( + this->_internal_begin()); } // optional uint32 end = 5; - if (has_end()) { + if (cached_has_bits & 0x00000010u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::UInt32Size( - this->end()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::UInt32Size( + this->_internal_end()); } } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void SentencePieceText_SentencePiece::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void SentencePieceText_SentencePiece::MergeFrom(const SentencePieceText_SentencePiece& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.SentencePieceText.SentencePiece) GOOGLE_DCHECK_NE(&from, this); _extensions_.MergeFrom(from._extensions_); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = from._has_bits_[0]; - if (cached_has_bits & 31u) { + if (cached_has_bits & 0x0000001fu) { if (cached_has_bits & 0x00000001u) { - set_has_piece(); - piece_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.piece_); + _internal_set_piece(from._internal_piece()); } if (cached_has_bits & 0x00000002u) { - set_has_surface(); - surface_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.surface_); + _internal_set_surface(from._internal_surface()); } if (cached_has_bits & 0x00000004u) { id_ = from.id_; @@ -434,25 +419,22 @@ bool SentencePieceText_SentencePiece::IsInitialized() const { return true; } -void SentencePieceText_SentencePiece::Swap(SentencePieceText_SentencePiece* other) { - if (other == this) return; - InternalSwap(other); -} void SentencePieceText_SentencePiece::InternalSwap(SentencePieceText_SentencePiece* other) { using std::swap; - piece_.Swap(&other->piece_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - surface_.Swap(&other->surface_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - swap(id_, other->id_); - swap(begin_, other->begin_); - swap(end_, other->end_); - swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); _extensions_.Swap(&other->_extensions_); + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(_has_bits_[0], other->_has_bits_[0]); + piece_.Swap(&other->piece_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + surface_.Swap(&other->surface_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(SentencePieceText_SentencePiece, end_) + + sizeof(SentencePieceText_SentencePiece::end_) + - PROTOBUF_FIELD_OFFSET(SentencePieceText_SentencePiece, id_)>( + reinterpret_cast(&id_), + reinterpret_cast(&other->id_)); } -::std::string SentencePieceText_SentencePiece::GetTypeName() const { +std::string SentencePieceText_SentencePiece::GetTypeName() const { return "sentencepiece.SentencePieceText.SentencePiece"; } @@ -461,60 +443,75 @@ ::std::string SentencePieceText_SentencePiece::GetTypeName() const { void SentencePieceText::InitAsDefaultInstance() { } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int SentencePieceText::kTextFieldNumber; -const int SentencePieceText::kPiecesFieldNumber; -const int SentencePieceText::kScoreFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 +class SentencePieceText::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static void set_has_text(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static void set_has_score(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } +}; -SentencePieceText::SentencePieceText() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_2eproto::scc_info_SentencePieceText.base); +SentencePieceText::SentencePieceText(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + _extensions_(arena), + pieces_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.SentencePieceText) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.SentencePieceText) } SentencePieceText::SentencePieceText(const SentencePieceText& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_), pieces_(from.pieces_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); _extensions_.MergeFrom(from._extensions_); - text_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_text()) { - text_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.text_); + text_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_text()) { + text_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_text(), + GetArena()); } score_ = from.score_; // @@protoc_insertion_point(copy_constructor:sentencepiece.SentencePieceText) } void SentencePieceText::SharedCtor() { - text_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SentencePieceText_sentencepiece_2eproto.base); + text_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); score_ = 0; } SentencePieceText::~SentencePieceText() { // @@protoc_insertion_point(destructor:sentencepiece.SentencePieceText) SharedDtor(); + _internal_metadata_.Delete(); } void SentencePieceText::SharedDtor() { - text_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + GOOGLE_DCHECK(GetArena() == nullptr); + text_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } +void SentencePieceText::ArenaDtor(void* object) { + SentencePieceText* _this = reinterpret_cast< SentencePieceText* >(object); + (void)_this; +} +void SentencePieceText::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void SentencePieceText::SetCachedSize(int size) const { _cached_size_.Set(size); } const SentencePieceText& SentencePieceText::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_2eproto::scc_info_SentencePieceText.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_SentencePieceText_sentencepiece_2eproto.base); return *internal_default_instance(); } void SentencePieceText::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.SentencePieceText) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; @@ -522,127 +519,116 @@ void SentencePieceText::Clear() { pieces_.Clear(); cached_has_bits = _has_bits_[0]; if (cached_has_bits & 0x00000001u) { - text_.ClearNonDefaultToEmptyNoArena(); + text_.ClearNonDefaultToEmpty(); } score_ = 0; _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool SentencePieceText::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.SentencePieceText) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + _internal_metadata_.Clear(); +} + +const char* SentencePieceText::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { // optional string text = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_text())); - } else { - goto handle_unusual; - } - break; - } - + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + auto str = _internal_mutable_text(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // repeated .sentencepiece.SentencePieceText.SentencePiece pieces = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, add_pieces())); - } else { - goto handle_unusual; - } - break; - } - + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(_internal_add_pieces(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); + } else goto handle_unusual; + continue; // optional float score = 3; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(29u /* 29 & 0xFF */)) { - set_has_score(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( - input, &score_))); - } else { - goto handle_unusual; - } - break; - } - + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 29)) { + _Internal::set_has_score(&has_bits); + score_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(float); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - if ((1600u <= tag)) { - DO_(_extensions_.ParseField(tag, input, - internal_default_instance(), - &unknown_fields_stream)); - continue; - } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + if ((1600u <= tag)) { + ptr = _extensions_.ParseField(tag, ptr, + internal_default_instance(), &_internal_metadata_, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.SentencePieceText) - return true; + _has_bits_.Or(has_bits); + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.SentencePieceText) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void SentencePieceText::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.SentencePieceText) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* SentencePieceText::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.SentencePieceText) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = _has_bits_[0]; // optional string text = 1; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->text(), output); + target = stream->WriteStringMaybeAliased( + 1, this->_internal_text(), target); } // repeated .sentencepiece.SentencePieceText.SentencePiece pieces = 2; for (unsigned int i = 0, - n = static_cast(this->pieces_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 2, - this->pieces(static_cast(i)), - output); + n = static_cast(this->_internal_pieces_size()); i < n; i++) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(2, this->_internal_pieces(i), target, stream); } // optional float score = 3; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteFloat(3, this->score(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(3, this->_internal_score(), target); } // Extension range [200, 536870912) - _extensions_.SerializeWithCachedSizes( - 200, 536870912, output); + target = _extensions_._InternalSerialize( + 200, 536870912, target, stream); - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.SentencePieceText) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.SentencePieceText) + return target; } size_t SentencePieceText::ByteSizeLong() const { @@ -651,57 +637,59 @@ size_t SentencePieceText::ByteSizeLong() const { total_size += _extensions_.ByteSize(); - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; // repeated .sentencepiece.SentencePieceText.SentencePiece pieces = 2; - { - unsigned int count = static_cast(this->pieces_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->pieces(static_cast(i))); - } + total_size += 1UL * this->_internal_pieces_size(); + for (const auto& msg : this->pieces_) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } - if (_has_bits_[0 / 32] & 3u) { + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x00000003u) { // optional string text = 1; - if (has_text()) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->text()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_text()); } // optional float score = 3; - if (has_score()) { + if (cached_has_bits & 0x00000002u) { total_size += 1 + 4; } } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void SentencePieceText::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void SentencePieceText::MergeFrom(const SentencePieceText& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.SentencePieceText) GOOGLE_DCHECK_NE(&from, this); _extensions_.MergeFrom(from._extensions_); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; pieces_.MergeFrom(from.pieces_); cached_has_bits = from._has_bits_[0]; - if (cached_has_bits & 3u) { + if (cached_has_bits & 0x00000003u) { if (cached_has_bits & 0x00000001u) { - set_has_text(); - text_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.text_); + _internal_set_text(from._internal_text()); } if (cached_has_bits & 0x00000002u) { score_ = from.score_; @@ -722,26 +710,21 @@ bool SentencePieceText::IsInitialized() const { return false; } - if (!::google::protobuf::internal::AllAreInitialized(this->pieces())) return false; + if (!::PROTOBUF_NAMESPACE_ID::internal::AllAreInitialized(pieces_)) return false; return true; } -void SentencePieceText::Swap(SentencePieceText* other) { - if (other == this) return; - InternalSwap(other); -} void SentencePieceText::InternalSwap(SentencePieceText* other) { using std::swap; - CastToBase(&pieces_)->InternalSwap(CastToBase(&other->pieces_)); - text_.Swap(&other->text_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - swap(score_, other->score_); - swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); _extensions_.Swap(&other->_extensions_); + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(_has_bits_[0], other->_has_bits_[0]); + pieces_.InternalSwap(&other->pieces_); + text_.Swap(&other->text_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + swap(score_, other->score_); } -::std::string SentencePieceText::GetTypeName() const { +std::string SentencePieceText::GetTypeName() const { return "sentencepiece.SentencePieceText"; } @@ -750,157 +733,161 @@ ::std::string SentencePieceText::GetTypeName() const { void NBestSentencePieceText::InitAsDefaultInstance() { } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int NBestSentencePieceText::kNbestsFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 +class NBestSentencePieceText::_Internal { + public: +}; -NBestSentencePieceText::NBestSentencePieceText() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_2eproto::scc_info_NBestSentencePieceText.base); +NBestSentencePieceText::NBestSentencePieceText(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + nbests_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.NBestSentencePieceText) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.NBestSentencePieceText) } NBestSentencePieceText::NBestSentencePieceText(const NBestSentencePieceText& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), - _has_bits_(from._has_bits_), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), nbests_(from.nbests_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:sentencepiece.NBestSentencePieceText) } void NBestSentencePieceText::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_NBestSentencePieceText_sentencepiece_2eproto.base); } NBestSentencePieceText::~NBestSentencePieceText() { // @@protoc_insertion_point(destructor:sentencepiece.NBestSentencePieceText) SharedDtor(); + _internal_metadata_.Delete(); } void NBestSentencePieceText::SharedDtor() { + GOOGLE_DCHECK(GetArena() == nullptr); } +void NBestSentencePieceText::ArenaDtor(void* object) { + NBestSentencePieceText* _this = reinterpret_cast< NBestSentencePieceText* >(object); + (void)_this; +} +void NBestSentencePieceText::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void NBestSentencePieceText::SetCachedSize(int size) const { _cached_size_.Set(size); } const NBestSentencePieceText& NBestSentencePieceText::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_2eproto::scc_info_NBestSentencePieceText.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_NBestSentencePieceText_sentencepiece_2eproto.base); return *internal_default_instance(); } void NBestSentencePieceText::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.NBestSentencePieceText) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; nbests_.Clear(); - _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool NBestSentencePieceText::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.NBestSentencePieceText) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // repeated .sentencepiece.SentencePieceText nbests = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, add_nbests())); - } else { - goto handle_unusual; - } - break; - } + _internal_metadata_.Clear(); +} +const char* NBestSentencePieceText::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // repeated .sentencepiece.SentencePieceText nbests = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(_internal_add_nbests(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.NBestSentencePieceText) - return true; + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.NBestSentencePieceText) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void NBestSentencePieceText::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.NBestSentencePieceText) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* NBestSentencePieceText::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.NBestSentencePieceText) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; // repeated .sentencepiece.SentencePieceText nbests = 1; for (unsigned int i = 0, - n = static_cast(this->nbests_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 1, - this->nbests(static_cast(i)), - output); + n = static_cast(this->_internal_nbests_size()); i < n; i++) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(1, this->_internal_nbests(i), target, stream); } - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.NBestSentencePieceText) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.NBestSentencePieceText) + return target; } size_t NBestSentencePieceText::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:sentencepiece.NBestSentencePieceText) size_t total_size = 0; - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; // repeated .sentencepiece.SentencePieceText nbests = 1; - { - unsigned int count = static_cast(this->nbests_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->nbests(static_cast(i))); - } + total_size += 1UL * this->_internal_nbests_size(); + for (const auto& msg : this->nbests_) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void NBestSentencePieceText::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void NBestSentencePieceText::MergeFrom(const NBestSentencePieceText& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.NBestSentencePieceText) GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; nbests_.MergeFrom(from.nbests_); @@ -914,40 +901,34 @@ void NBestSentencePieceText::CopyFrom(const NBestSentencePieceText& from) { } bool NBestSentencePieceText::IsInitialized() const { - if (!::google::protobuf::internal::AllAreInitialized(this->nbests())) return false; + if (!::PROTOBUF_NAMESPACE_ID::internal::AllAreInitialized(nbests_)) return false; return true; } -void NBestSentencePieceText::Swap(NBestSentencePieceText* other) { - if (other == this) return; - InternalSwap(other); -} void NBestSentencePieceText::InternalSwap(NBestSentencePieceText* other) { using std::swap; - CastToBase(&nbests_)->InternalSwap(CastToBase(&other->nbests_)); - swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); + _internal_metadata_.Swap(&other->_internal_metadata_); + nbests_.InternalSwap(&other->nbests_); } -::std::string NBestSentencePieceText::GetTypeName() const { +std::string NBestSentencePieceText::GetTypeName() const { return "sentencepiece.NBestSentencePieceText"; } // @@protoc_insertion_point(namespace_scope) } // namespace sentencepiece -namespace google { -namespace protobuf { -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::SentencePieceText_SentencePiece* Arena::CreateMaybeMessage< ::sentencepiece::SentencePieceText_SentencePiece >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::SentencePieceText_SentencePiece >(arena); +PROTOBUF_NAMESPACE_OPEN +template<> PROTOBUF_NOINLINE ::sentencepiece::SentencePieceText_SentencePiece* Arena::CreateMaybeMessage< ::sentencepiece::SentencePieceText_SentencePiece >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::SentencePieceText_SentencePiece >(arena); } -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::SentencePieceText* Arena::CreateMaybeMessage< ::sentencepiece::SentencePieceText >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::SentencePieceText >(arena); +template<> PROTOBUF_NOINLINE ::sentencepiece::SentencePieceText* Arena::CreateMaybeMessage< ::sentencepiece::SentencePieceText >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::SentencePieceText >(arena); } -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::NBestSentencePieceText* Arena::CreateMaybeMessage< ::sentencepiece::NBestSentencePieceText >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::NBestSentencePieceText >(arena); +template<> PROTOBUF_NOINLINE ::sentencepiece::NBestSentencePieceText* Arena::CreateMaybeMessage< ::sentencepiece::NBestSentencePieceText >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::NBestSentencePieceText >(arena); } -} // namespace protobuf -} // namespace google +PROTOBUF_NAMESPACE_CLOSE // @@protoc_insertion_point(global_scope) +#include diff --git a/src/builtin_pb/sentencepiece.pb.h b/src/builtin_pb/sentencepiece.pb.h index 91dc85d9..8352d7a3 100644 --- a/src/builtin_pb/sentencepiece.pb.h +++ b/src/builtin_pb/sentencepiece.pb.h @@ -1,24 +1,25 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: sentencepiece.proto -#ifndef PROTOBUF_INCLUDED_sentencepiece_2eproto -#define PROTOBUF_INCLUDED_sentencepiece_2eproto +#ifndef GOOGLE_PROTOBUF_INCLUDED_sentencepiece_2eproto +#define GOOGLE_PROTOBUF_INCLUDED_sentencepiece_2eproto +#include #include -#include - -#if GOOGLE_PROTOBUF_VERSION < 3006001 +#include +#if PROTOBUF_VERSION < 3012000 #error This file was generated by a newer version of protoc which is -#error incompatible with your Protocol Buffer headers. Please update +#error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3006001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is -#error incompatible with your Protocol Buffer headers. Please +#error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. #endif +#include #include #include #include @@ -30,19 +31,26 @@ #include // IWYU pragma: export #include // IWYU pragma: export // @@protoc_insertion_point(includes) -#define PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_2eproto +#include +#define PROTOBUF_INTERNAL_EXPORT_sentencepiece_2eproto +PROTOBUF_NAMESPACE_OPEN +namespace internal { +class AnyMetadata; +} // namespace internal +PROTOBUF_NAMESPACE_CLOSE -namespace protobuf_sentencepiece_2eproto { // Internal implementation detail -- do not use these members. -struct TableStruct { - static const ::google::protobuf::internal::ParseTableField entries[]; - static const ::google::protobuf::internal::AuxillaryParseTableField aux[]; - static const ::google::protobuf::internal::ParseTable schema[3]; - static const ::google::protobuf::internal::FieldMetadata field_metadata[]; - static const ::google::protobuf::internal::SerializationTable serialization_table[]; - static const ::google::protobuf::uint32 offsets[]; +struct TableStruct_sentencepiece_2eproto { + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[3] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; + static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; + static const ::PROTOBUF_NAMESPACE_ID::uint32 offsets[]; }; -} // namespace protobuf_sentencepiece_2eproto namespace sentencepiece { class NBestSentencePieceText; class NBestSentencePieceTextDefaultTypeInternal; @@ -54,48 +62,45 @@ class SentencePieceText_SentencePiece; class SentencePieceText_SentencePieceDefaultTypeInternal; extern SentencePieceText_SentencePieceDefaultTypeInternal _SentencePieceText_SentencePiece_default_instance_; } // namespace sentencepiece -namespace google { -namespace protobuf { +PROTOBUF_NAMESPACE_OPEN template<> ::sentencepiece::NBestSentencePieceText* Arena::CreateMaybeMessage<::sentencepiece::NBestSentencePieceText>(Arena*); template<> ::sentencepiece::SentencePieceText* Arena::CreateMaybeMessage<::sentencepiece::SentencePieceText>(Arena*); template<> ::sentencepiece::SentencePieceText_SentencePiece* Arena::CreateMaybeMessage<::sentencepiece::SentencePieceText_SentencePiece>(Arena*); -} // namespace protobuf -} // namespace google +PROTOBUF_NAMESPACE_CLOSE namespace sentencepiece { // =================================================================== -class SentencePieceText_SentencePiece : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SentencePieceText.SentencePiece) */ { +class SentencePieceText_SentencePiece PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SentencePieceText.SentencePiece) */ { public: - SentencePieceText_SentencePiece(); + inline SentencePieceText_SentencePiece() : SentencePieceText_SentencePiece(nullptr) {}; virtual ~SentencePieceText_SentencePiece(); SentencePieceText_SentencePiece(const SentencePieceText_SentencePiece& from); - - inline SentencePieceText_SentencePiece& operator=(const SentencePieceText_SentencePiece& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 SentencePieceText_SentencePiece(SentencePieceText_SentencePiece&& from) noexcept : SentencePieceText_SentencePiece() { *this = ::std::move(from); } + inline SentencePieceText_SentencePiece& operator=(const SentencePieceText_SentencePiece& from) { + CopyFrom(from); + return *this; + } inline SentencePieceText_SentencePiece& operator=(SentencePieceText_SentencePiece&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const SentencePieceText_SentencePiece& default_instance(); @@ -108,165 +113,223 @@ class SentencePieceText_SentencePiece : public ::google::protobuf::MessageLite / static constexpr int kIndexInFileMessages = 0; - void Swap(SentencePieceText_SentencePiece* other); friend void swap(SentencePieceText_SentencePiece& a, SentencePieceText_SentencePiece& b) { a.Swap(&b); } + inline void Swap(SentencePieceText_SentencePiece* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(SentencePieceText_SentencePiece* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline SentencePieceText_SentencePiece* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - SentencePieceText_SentencePiece* New(::google::protobuf::Arena* arena) const final { + SentencePieceText_SentencePiece* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const SentencePieceText_SentencePiece& from); void MergeFrom(const SentencePieceText_SentencePiece& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(SentencePieceText_SentencePiece* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.SentencePieceText.SentencePiece"; } + protected: + explicit SentencePieceText_SentencePiece(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- + enum : int { + kPieceFieldNumber = 1, + kSurfaceFieldNumber = 3, + kIdFieldNumber = 2, + kBeginFieldNumber = 4, + kEndFieldNumber = 5, + }; // optional string piece = 1; bool has_piece() const; + private: + bool _internal_has_piece() const; + public: void clear_piece(); - static const int kPieceFieldNumber = 1; - const ::std::string& piece() const; - void set_piece(const ::std::string& value); - #if LANG_CXX11 - void set_piece(::std::string&& value); - #endif + const std::string& piece() const; + void set_piece(const std::string& value); + void set_piece(std::string&& value); void set_piece(const char* value); void set_piece(const char* value, size_t size); - ::std::string* mutable_piece(); - ::std::string* release_piece(); - void set_allocated_piece(::std::string* piece); + std::string* mutable_piece(); + std::string* release_piece(); + void set_allocated_piece(std::string* piece); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_piece(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_piece( + std::string* piece); + private: + const std::string& _internal_piece() const; + void _internal_set_piece(const std::string& value); + std::string* _internal_mutable_piece(); + public: // optional string surface = 3; bool has_surface() const; + private: + bool _internal_has_surface() const; + public: void clear_surface(); - static const int kSurfaceFieldNumber = 3; - const ::std::string& surface() const; - void set_surface(const ::std::string& value); - #if LANG_CXX11 - void set_surface(::std::string&& value); - #endif + const std::string& surface() const; + void set_surface(const std::string& value); + void set_surface(std::string&& value); void set_surface(const char* value); void set_surface(const char* value, size_t size); - ::std::string* mutable_surface(); - ::std::string* release_surface(); - void set_allocated_surface(::std::string* surface); + std::string* mutable_surface(); + std::string* release_surface(); + void set_allocated_surface(std::string* surface); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_surface(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_surface( + std::string* surface); + private: + const std::string& _internal_surface() const; + void _internal_set_surface(const std::string& value); + std::string* _internal_mutable_surface(); + public: // optional uint32 id = 2; bool has_id() const; + private: + bool _internal_has_id() const; + public: void clear_id(); - static const int kIdFieldNumber = 2; - ::google::protobuf::uint32 id() const; - void set_id(::google::protobuf::uint32 value); + ::PROTOBUF_NAMESPACE_ID::uint32 id() const; + void set_id(::PROTOBUF_NAMESPACE_ID::uint32 value); + private: + ::PROTOBUF_NAMESPACE_ID::uint32 _internal_id() const; + void _internal_set_id(::PROTOBUF_NAMESPACE_ID::uint32 value); + public: // optional uint32 begin = 4; bool has_begin() const; + private: + bool _internal_has_begin() const; + public: void clear_begin(); - static const int kBeginFieldNumber = 4; - ::google::protobuf::uint32 begin() const; - void set_begin(::google::protobuf::uint32 value); + ::PROTOBUF_NAMESPACE_ID::uint32 begin() const; + void set_begin(::PROTOBUF_NAMESPACE_ID::uint32 value); + private: + ::PROTOBUF_NAMESPACE_ID::uint32 _internal_begin() const; + void _internal_set_begin(::PROTOBUF_NAMESPACE_ID::uint32 value); + public: // optional uint32 end = 5; bool has_end() const; + private: + bool _internal_has_end() const; + public: void clear_end(); - static const int kEndFieldNumber = 5; - ::google::protobuf::uint32 end() const; - void set_end(::google::protobuf::uint32 value); + ::PROTOBUF_NAMESPACE_ID::uint32 end() const; + void set_end(::PROTOBUF_NAMESPACE_ID::uint32 value); + private: + ::PROTOBUF_NAMESPACE_ID::uint32 _internal_end() const; + void _internal_set_end(::PROTOBUF_NAMESPACE_ID::uint32 value); + public: GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(SentencePieceText_SentencePiece) // @@protoc_insertion_point(class_scope:sentencepiece.SentencePieceText.SentencePiece) private: - void set_has_piece(); - void clear_has_piece(); - void set_has_id(); - void clear_has_id(); - void set_has_surface(); - void clear_has_surface(); - void set_has_begin(); - void clear_has_begin(); - void set_has_end(); - void clear_has_end(); - - ::google::protobuf::internal::ExtensionSet _extensions_; - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::internal::ArenaStringPtr piece_; - ::google::protobuf::internal::ArenaStringPtr surface_; - ::google::protobuf::uint32 id_; - ::google::protobuf::uint32 begin_; - ::google::protobuf::uint32 end_; - friend struct ::protobuf_sentencepiece_2eproto::TableStruct; + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::ExtensionSet _extensions_; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr piece_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr surface_; + ::PROTOBUF_NAMESPACE_ID::uint32 id_; + ::PROTOBUF_NAMESPACE_ID::uint32 begin_; + ::PROTOBUF_NAMESPACE_ID::uint32 end_; + friend struct ::TableStruct_sentencepiece_2eproto; }; // ------------------------------------------------------------------- -class SentencePieceText : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SentencePieceText) */ { +class SentencePieceText PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SentencePieceText) */ { public: - SentencePieceText(); + inline SentencePieceText() : SentencePieceText(nullptr) {}; virtual ~SentencePieceText(); SentencePieceText(const SentencePieceText& from); - - inline SentencePieceText& operator=(const SentencePieceText& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 SentencePieceText(SentencePieceText&& from) noexcept : SentencePieceText() { *this = ::std::move(from); } + inline SentencePieceText& operator=(const SentencePieceText& from) { + CopyFrom(from); + return *this; + } inline SentencePieceText& operator=(SentencePieceText&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const SentencePieceText& default_instance(); @@ -279,50 +342,63 @@ class SentencePieceText : public ::google::protobuf::MessageLite /* @@protoc_ins static constexpr int kIndexInFileMessages = 1; - void Swap(SentencePieceText* other); friend void swap(SentencePieceText& a, SentencePieceText& b) { a.Swap(&b); } + inline void Swap(SentencePieceText* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(SentencePieceText* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline SentencePieceText* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - SentencePieceText* New(::google::protobuf::Arena* arena) const final { + SentencePieceText* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const SentencePieceText& from); void MergeFrom(const SentencePieceText& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(SentencePieceText* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.SentencePieceText"; } + protected: + explicit SentencePieceText(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- @@ -330,91 +406,120 @@ class SentencePieceText : public ::google::protobuf::MessageLite /* @@protoc_ins // accessors ------------------------------------------------------- + enum : int { + kPiecesFieldNumber = 2, + kTextFieldNumber = 1, + kScoreFieldNumber = 3, + }; // repeated .sentencepiece.SentencePieceText.SentencePiece pieces = 2; int pieces_size() const; + private: + int _internal_pieces_size() const; + public: void clear_pieces(); - static const int kPiecesFieldNumber = 2; ::sentencepiece::SentencePieceText_SentencePiece* mutable_pieces(int index); - ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >* + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >* mutable_pieces(); + private: + const ::sentencepiece::SentencePieceText_SentencePiece& _internal_pieces(int index) const; + ::sentencepiece::SentencePieceText_SentencePiece* _internal_add_pieces(); + public: const ::sentencepiece::SentencePieceText_SentencePiece& pieces(int index) const; ::sentencepiece::SentencePieceText_SentencePiece* add_pieces(); - const ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >& + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >& pieces() const; // optional string text = 1; bool has_text() const; + private: + bool _internal_has_text() const; + public: void clear_text(); - static const int kTextFieldNumber = 1; - const ::std::string& text() const; - void set_text(const ::std::string& value); - #if LANG_CXX11 - void set_text(::std::string&& value); - #endif + const std::string& text() const; + void set_text(const std::string& value); + void set_text(std::string&& value); void set_text(const char* value); void set_text(const char* value, size_t size); - ::std::string* mutable_text(); - ::std::string* release_text(); - void set_allocated_text(::std::string* text); + std::string* mutable_text(); + std::string* release_text(); + void set_allocated_text(std::string* text); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_text(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_text( + std::string* text); + private: + const std::string& _internal_text() const; + void _internal_set_text(const std::string& value); + std::string* _internal_mutable_text(); + public: // optional float score = 3; bool has_score() const; + private: + bool _internal_has_score() const; + public: void clear_score(); - static const int kScoreFieldNumber = 3; float score() const; void set_score(float value); + private: + float _internal_score() const; + void _internal_set_score(float value); + public: GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(SentencePieceText) // @@protoc_insertion_point(class_scope:sentencepiece.SentencePieceText) private: - void set_has_text(); - void clear_has_text(); - void set_has_score(); - void clear_has_score(); - - ::google::protobuf::internal::ExtensionSet _extensions_; - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece > pieces_; - ::google::protobuf::internal::ArenaStringPtr text_; + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::ExtensionSet _extensions_; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece > pieces_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr text_; float score_; - friend struct ::protobuf_sentencepiece_2eproto::TableStruct; + friend struct ::TableStruct_sentencepiece_2eproto; }; // ------------------------------------------------------------------- -class NBestSentencePieceText : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.NBestSentencePieceText) */ { +class NBestSentencePieceText PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.NBestSentencePieceText) */ { public: - NBestSentencePieceText(); + inline NBestSentencePieceText() : NBestSentencePieceText(nullptr) {}; virtual ~NBestSentencePieceText(); NBestSentencePieceText(const NBestSentencePieceText& from); - - inline NBestSentencePieceText& operator=(const NBestSentencePieceText& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 NBestSentencePieceText(NBestSentencePieceText&& from) noexcept : NBestSentencePieceText() { *this = ::std::move(from); } + inline NBestSentencePieceText& operator=(const NBestSentencePieceText& from) { + CopyFrom(from); + return *this; + } inline NBestSentencePieceText& operator=(NBestSentencePieceText&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const NBestSentencePieceText& default_instance(); @@ -427,75 +532,99 @@ class NBestSentencePieceText : public ::google::protobuf::MessageLite /* @@proto static constexpr int kIndexInFileMessages = 2; - void Swap(NBestSentencePieceText* other); friend void swap(NBestSentencePieceText& a, NBestSentencePieceText& b) { a.Swap(&b); } + inline void Swap(NBestSentencePieceText* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(NBestSentencePieceText* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline NBestSentencePieceText* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - NBestSentencePieceText* New(::google::protobuf::Arena* arena) const final { + NBestSentencePieceText* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const NBestSentencePieceText& from); void MergeFrom(const NBestSentencePieceText& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(NBestSentencePieceText* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.NBestSentencePieceText"; } + protected: + explicit NBestSentencePieceText(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- + enum : int { + kNbestsFieldNumber = 1, + }; // repeated .sentencepiece.SentencePieceText nbests = 1; int nbests_size() const; + private: + int _internal_nbests_size() const; + public: void clear_nbests(); - static const int kNbestsFieldNumber = 1; ::sentencepiece::SentencePieceText* mutable_nbests(int index); - ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText >* + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText >* mutable_nbests(); + private: + const ::sentencepiece::SentencePieceText& _internal_nbests(int index) const; + ::sentencepiece::SentencePieceText* _internal_add_nbests(); + public: const ::sentencepiece::SentencePieceText& nbests(int index) const; ::sentencepiece::SentencePieceText* add_nbests(); - const ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText >& + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText >& nbests() const; // @@protoc_insertion_point(class_scope:sentencepiece.NBestSentencePieceText) private: - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText > nbests_; - friend struct ::protobuf_sentencepiece_2eproto::TableStruct; + class _Internal; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText > nbests_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_sentencepiece_2eproto; }; // =================================================================== @@ -509,206 +638,272 @@ class NBestSentencePieceText : public ::google::protobuf::MessageLite /* @@proto // SentencePieceText_SentencePiece // optional string piece = 1; -inline bool SentencePieceText_SentencePiece::has_piece() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void SentencePieceText_SentencePiece::set_has_piece() { - _has_bits_[0] |= 0x00000001u; +inline bool SentencePieceText_SentencePiece::_internal_has_piece() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + return value; } -inline void SentencePieceText_SentencePiece::clear_has_piece() { - _has_bits_[0] &= ~0x00000001u; +inline bool SentencePieceText_SentencePiece::has_piece() const { + return _internal_has_piece(); } inline void SentencePieceText_SentencePiece::clear_piece() { - piece_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_piece(); + piece_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000001u; } -inline const ::std::string& SentencePieceText_SentencePiece::piece() const { +inline const std::string& SentencePieceText_SentencePiece::piece() const { // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.piece) - return piece_.GetNoArena(); + return _internal_piece(); } -inline void SentencePieceText_SentencePiece::set_piece(const ::std::string& value) { - set_has_piece(); - piece_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void SentencePieceText_SentencePiece::set_piece(const std::string& value) { + _internal_set_piece(value); // @@protoc_insertion_point(field_set:sentencepiece.SentencePieceText.SentencePiece.piece) } -#if LANG_CXX11 -inline void SentencePieceText_SentencePiece::set_piece(::std::string&& value) { - set_has_piece(); - piece_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* SentencePieceText_SentencePiece::mutable_piece() { + // @@protoc_insertion_point(field_mutable:sentencepiece.SentencePieceText.SentencePiece.piece) + return _internal_mutable_piece(); +} +inline const std::string& SentencePieceText_SentencePiece::_internal_piece() const { + return piece_.Get(); +} +inline void SentencePieceText_SentencePiece::_internal_set_piece(const std::string& value) { + _has_bits_[0] |= 0x00000001u; + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void SentencePieceText_SentencePiece::set_piece(std::string&& value) { + _has_bits_[0] |= 0x00000001u; + piece_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.SentencePieceText.SentencePiece.piece) } -#endif inline void SentencePieceText_SentencePiece::set_piece(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_piece(); - piece_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000001u; + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.SentencePieceText.SentencePiece.piece) } -inline void SentencePieceText_SentencePiece::set_piece(const char* value, size_t size) { - set_has_piece(); - piece_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void SentencePieceText_SentencePiece::set_piece(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000001u; + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.SentencePieceText.SentencePiece.piece) } -inline ::std::string* SentencePieceText_SentencePiece::mutable_piece() { - set_has_piece(); - // @@protoc_insertion_point(field_mutable:sentencepiece.SentencePieceText.SentencePiece.piece) - return piece_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* SentencePieceText_SentencePiece::_internal_mutable_piece() { + _has_bits_[0] |= 0x00000001u; + return piece_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* SentencePieceText_SentencePiece::release_piece() { +inline std::string* SentencePieceText_SentencePiece::release_piece() { // @@protoc_insertion_point(field_release:sentencepiece.SentencePieceText.SentencePiece.piece) - if (!has_piece()) { - return NULL; + if (!_internal_has_piece()) { + return nullptr; } - clear_has_piece(); - return piece_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000001u; + return piece_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void SentencePieceText_SentencePiece::set_allocated_piece(::std::string* piece) { - if (piece != NULL) { - set_has_piece(); +inline void SentencePieceText_SentencePiece::set_allocated_piece(std::string* piece) { + if (piece != nullptr) { + _has_bits_[0] |= 0x00000001u; } else { - clear_has_piece(); + _has_bits_[0] &= ~0x00000001u; } - piece_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), piece); + piece_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), piece, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.SentencePieceText.SentencePiece.piece) } +inline std::string* SentencePieceText_SentencePiece::unsafe_arena_release_piece() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.SentencePieceText.SentencePiece.piece) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000001u; + return piece_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void SentencePieceText_SentencePiece::unsafe_arena_set_allocated_piece( + std::string* piece) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (piece != nullptr) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + piece_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + piece, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.SentencePieceText.SentencePiece.piece) +} // optional uint32 id = 2; -inline bool SentencePieceText_SentencePiece::has_id() const { - return (_has_bits_[0] & 0x00000004u) != 0; +inline bool SentencePieceText_SentencePiece::_internal_has_id() const { + bool value = (_has_bits_[0] & 0x00000004u) != 0; + return value; } -inline void SentencePieceText_SentencePiece::set_has_id() { - _has_bits_[0] |= 0x00000004u; -} -inline void SentencePieceText_SentencePiece::clear_has_id() { - _has_bits_[0] &= ~0x00000004u; +inline bool SentencePieceText_SentencePiece::has_id() const { + return _internal_has_id(); } inline void SentencePieceText_SentencePiece::clear_id() { id_ = 0u; - clear_has_id(); + _has_bits_[0] &= ~0x00000004u; } -inline ::google::protobuf::uint32 SentencePieceText_SentencePiece::id() const { - // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.id) +inline ::PROTOBUF_NAMESPACE_ID::uint32 SentencePieceText_SentencePiece::_internal_id() const { return id_; } -inline void SentencePieceText_SentencePiece::set_id(::google::protobuf::uint32 value) { - set_has_id(); +inline ::PROTOBUF_NAMESPACE_ID::uint32 SentencePieceText_SentencePiece::id() const { + // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.id) + return _internal_id(); +} +inline void SentencePieceText_SentencePiece::_internal_set_id(::PROTOBUF_NAMESPACE_ID::uint32 value) { + _has_bits_[0] |= 0x00000004u; id_ = value; +} +inline void SentencePieceText_SentencePiece::set_id(::PROTOBUF_NAMESPACE_ID::uint32 value) { + _internal_set_id(value); // @@protoc_insertion_point(field_set:sentencepiece.SentencePieceText.SentencePiece.id) } // optional string surface = 3; -inline bool SentencePieceText_SentencePiece::has_surface() const { - return (_has_bits_[0] & 0x00000002u) != 0; +inline bool SentencePieceText_SentencePiece::_internal_has_surface() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + return value; } -inline void SentencePieceText_SentencePiece::set_has_surface() { - _has_bits_[0] |= 0x00000002u; -} -inline void SentencePieceText_SentencePiece::clear_has_surface() { - _has_bits_[0] &= ~0x00000002u; +inline bool SentencePieceText_SentencePiece::has_surface() const { + return _internal_has_surface(); } inline void SentencePieceText_SentencePiece::clear_surface() { - surface_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_surface(); + surface_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000002u; } -inline const ::std::string& SentencePieceText_SentencePiece::surface() const { +inline const std::string& SentencePieceText_SentencePiece::surface() const { // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.surface) - return surface_.GetNoArena(); + return _internal_surface(); } -inline void SentencePieceText_SentencePiece::set_surface(const ::std::string& value) { - set_has_surface(); - surface_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void SentencePieceText_SentencePiece::set_surface(const std::string& value) { + _internal_set_surface(value); // @@protoc_insertion_point(field_set:sentencepiece.SentencePieceText.SentencePiece.surface) } -#if LANG_CXX11 -inline void SentencePieceText_SentencePiece::set_surface(::std::string&& value) { - set_has_surface(); - surface_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* SentencePieceText_SentencePiece::mutable_surface() { + // @@protoc_insertion_point(field_mutable:sentencepiece.SentencePieceText.SentencePiece.surface) + return _internal_mutable_surface(); +} +inline const std::string& SentencePieceText_SentencePiece::_internal_surface() const { + return surface_.Get(); +} +inline void SentencePieceText_SentencePiece::_internal_set_surface(const std::string& value) { + _has_bits_[0] |= 0x00000002u; + surface_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void SentencePieceText_SentencePiece::set_surface(std::string&& value) { + _has_bits_[0] |= 0x00000002u; + surface_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.SentencePieceText.SentencePiece.surface) } -#endif inline void SentencePieceText_SentencePiece::set_surface(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_surface(); - surface_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000002u; + surface_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.SentencePieceText.SentencePiece.surface) } -inline void SentencePieceText_SentencePiece::set_surface(const char* value, size_t size) { - set_has_surface(); - surface_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void SentencePieceText_SentencePiece::set_surface(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000002u; + surface_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.SentencePieceText.SentencePiece.surface) } -inline ::std::string* SentencePieceText_SentencePiece::mutable_surface() { - set_has_surface(); - // @@protoc_insertion_point(field_mutable:sentencepiece.SentencePieceText.SentencePiece.surface) - return surface_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* SentencePieceText_SentencePiece::_internal_mutable_surface() { + _has_bits_[0] |= 0x00000002u; + return surface_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* SentencePieceText_SentencePiece::release_surface() { +inline std::string* SentencePieceText_SentencePiece::release_surface() { // @@protoc_insertion_point(field_release:sentencepiece.SentencePieceText.SentencePiece.surface) - if (!has_surface()) { - return NULL; + if (!_internal_has_surface()) { + return nullptr; } - clear_has_surface(); - return surface_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000002u; + return surface_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void SentencePieceText_SentencePiece::set_allocated_surface(::std::string* surface) { - if (surface != NULL) { - set_has_surface(); +inline void SentencePieceText_SentencePiece::set_allocated_surface(std::string* surface) { + if (surface != nullptr) { + _has_bits_[0] |= 0x00000002u; } else { - clear_has_surface(); + _has_bits_[0] &= ~0x00000002u; } - surface_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), surface); + surface_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), surface, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.SentencePieceText.SentencePiece.surface) } +inline std::string* SentencePieceText_SentencePiece::unsafe_arena_release_surface() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.SentencePieceText.SentencePiece.surface) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000002u; + return surface_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void SentencePieceText_SentencePiece::unsafe_arena_set_allocated_surface( + std::string* surface) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (surface != nullptr) { + _has_bits_[0] |= 0x00000002u; + } else { + _has_bits_[0] &= ~0x00000002u; + } + surface_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + surface, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.SentencePieceText.SentencePiece.surface) +} // optional uint32 begin = 4; -inline bool SentencePieceText_SentencePiece::has_begin() const { - return (_has_bits_[0] & 0x00000008u) != 0; -} -inline void SentencePieceText_SentencePiece::set_has_begin() { - _has_bits_[0] |= 0x00000008u; +inline bool SentencePieceText_SentencePiece::_internal_has_begin() const { + bool value = (_has_bits_[0] & 0x00000008u) != 0; + return value; } -inline void SentencePieceText_SentencePiece::clear_has_begin() { - _has_bits_[0] &= ~0x00000008u; +inline bool SentencePieceText_SentencePiece::has_begin() const { + return _internal_has_begin(); } inline void SentencePieceText_SentencePiece::clear_begin() { begin_ = 0u; - clear_has_begin(); + _has_bits_[0] &= ~0x00000008u; } -inline ::google::protobuf::uint32 SentencePieceText_SentencePiece::begin() const { - // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.begin) +inline ::PROTOBUF_NAMESPACE_ID::uint32 SentencePieceText_SentencePiece::_internal_begin() const { return begin_; } -inline void SentencePieceText_SentencePiece::set_begin(::google::protobuf::uint32 value) { - set_has_begin(); +inline ::PROTOBUF_NAMESPACE_ID::uint32 SentencePieceText_SentencePiece::begin() const { + // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.begin) + return _internal_begin(); +} +inline void SentencePieceText_SentencePiece::_internal_set_begin(::PROTOBUF_NAMESPACE_ID::uint32 value) { + _has_bits_[0] |= 0x00000008u; begin_ = value; +} +inline void SentencePieceText_SentencePiece::set_begin(::PROTOBUF_NAMESPACE_ID::uint32 value) { + _internal_set_begin(value); // @@protoc_insertion_point(field_set:sentencepiece.SentencePieceText.SentencePiece.begin) } // optional uint32 end = 5; -inline bool SentencePieceText_SentencePiece::has_end() const { - return (_has_bits_[0] & 0x00000010u) != 0; -} -inline void SentencePieceText_SentencePiece::set_has_end() { - _has_bits_[0] |= 0x00000010u; +inline bool SentencePieceText_SentencePiece::_internal_has_end() const { + bool value = (_has_bits_[0] & 0x00000010u) != 0; + return value; } -inline void SentencePieceText_SentencePiece::clear_has_end() { - _has_bits_[0] &= ~0x00000010u; +inline bool SentencePieceText_SentencePiece::has_end() const { + return _internal_has_end(); } inline void SentencePieceText_SentencePiece::clear_end() { end_ = 0u; - clear_has_end(); + _has_bits_[0] &= ~0x00000010u; } -inline ::google::protobuf::uint32 SentencePieceText_SentencePiece::end() const { - // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.end) +inline ::PROTOBUF_NAMESPACE_ID::uint32 SentencePieceText_SentencePiece::_internal_end() const { return end_; } -inline void SentencePieceText_SentencePiece::set_end(::google::protobuf::uint32 value) { - set_has_end(); +inline ::PROTOBUF_NAMESPACE_ID::uint32 SentencePieceText_SentencePiece::end() const { + // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.SentencePiece.end) + return _internal_end(); +} +inline void SentencePieceText_SentencePiece::_internal_set_end(::PROTOBUF_NAMESPACE_ID::uint32 value) { + _has_bits_[0] |= 0x00000010u; end_ = value; +} +inline void SentencePieceText_SentencePiece::set_end(::PROTOBUF_NAMESPACE_ID::uint32 value) { + _internal_set_end(value); // @@protoc_insertion_point(field_set:sentencepiece.SentencePieceText.SentencePiece.end) } @@ -717,75 +912,105 @@ inline void SentencePieceText_SentencePiece::set_end(::google::protobuf::uint32 // SentencePieceText // optional string text = 1; -inline bool SentencePieceText::has_text() const { - return (_has_bits_[0] & 0x00000001u) != 0; +inline bool SentencePieceText::_internal_has_text() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + return value; } -inline void SentencePieceText::set_has_text() { - _has_bits_[0] |= 0x00000001u; -} -inline void SentencePieceText::clear_has_text() { - _has_bits_[0] &= ~0x00000001u; +inline bool SentencePieceText::has_text() const { + return _internal_has_text(); } inline void SentencePieceText::clear_text() { - text_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_text(); + text_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000001u; } -inline const ::std::string& SentencePieceText::text() const { +inline const std::string& SentencePieceText::text() const { // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.text) - return text_.GetNoArena(); + return _internal_text(); } -inline void SentencePieceText::set_text(const ::std::string& value) { - set_has_text(); - text_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void SentencePieceText::set_text(const std::string& value) { + _internal_set_text(value); // @@protoc_insertion_point(field_set:sentencepiece.SentencePieceText.text) } -#if LANG_CXX11 -inline void SentencePieceText::set_text(::std::string&& value) { - set_has_text(); - text_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* SentencePieceText::mutable_text() { + // @@protoc_insertion_point(field_mutable:sentencepiece.SentencePieceText.text) + return _internal_mutable_text(); +} +inline const std::string& SentencePieceText::_internal_text() const { + return text_.Get(); +} +inline void SentencePieceText::_internal_set_text(const std::string& value) { + _has_bits_[0] |= 0x00000001u; + text_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void SentencePieceText::set_text(std::string&& value) { + _has_bits_[0] |= 0x00000001u; + text_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.SentencePieceText.text) } -#endif inline void SentencePieceText::set_text(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_text(); - text_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000001u; + text_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.SentencePieceText.text) } -inline void SentencePieceText::set_text(const char* value, size_t size) { - set_has_text(); - text_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void SentencePieceText::set_text(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000001u; + text_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.SentencePieceText.text) } -inline ::std::string* SentencePieceText::mutable_text() { - set_has_text(); - // @@protoc_insertion_point(field_mutable:sentencepiece.SentencePieceText.text) - return text_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* SentencePieceText::_internal_mutable_text() { + _has_bits_[0] |= 0x00000001u; + return text_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* SentencePieceText::release_text() { +inline std::string* SentencePieceText::release_text() { // @@protoc_insertion_point(field_release:sentencepiece.SentencePieceText.text) - if (!has_text()) { - return NULL; + if (!_internal_has_text()) { + return nullptr; } - clear_has_text(); - return text_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000001u; + return text_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void SentencePieceText::set_allocated_text(::std::string* text) { - if (text != NULL) { - set_has_text(); +inline void SentencePieceText::set_allocated_text(std::string* text) { + if (text != nullptr) { + _has_bits_[0] |= 0x00000001u; } else { - clear_has_text(); + _has_bits_[0] &= ~0x00000001u; } - text_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), text); + text_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), text, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.SentencePieceText.text) } +inline std::string* SentencePieceText::unsafe_arena_release_text() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.SentencePieceText.text) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000001u; + return text_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void SentencePieceText::unsafe_arena_set_allocated_text( + std::string* text) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (text != nullptr) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + text_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + text, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.SentencePieceText.text) +} // repeated .sentencepiece.SentencePieceText.SentencePiece pieces = 2; -inline int SentencePieceText::pieces_size() const { +inline int SentencePieceText::_internal_pieces_size() const { return pieces_.size(); } +inline int SentencePieceText::pieces_size() const { + return _internal_pieces_size(); +} inline void SentencePieceText::clear_pieces() { pieces_.Clear(); } @@ -793,46 +1018,56 @@ inline ::sentencepiece::SentencePieceText_SentencePiece* SentencePieceText::muta // @@protoc_insertion_point(field_mutable:sentencepiece.SentencePieceText.pieces) return pieces_.Mutable(index); } -inline ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >* SentencePieceText::mutable_pieces() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.SentencePieceText.pieces) return &pieces_; } +inline const ::sentencepiece::SentencePieceText_SentencePiece& SentencePieceText::_internal_pieces(int index) const { + return pieces_.Get(index); +} inline const ::sentencepiece::SentencePieceText_SentencePiece& SentencePieceText::pieces(int index) const { // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.pieces) - return pieces_.Get(index); + return _internal_pieces(index); +} +inline ::sentencepiece::SentencePieceText_SentencePiece* SentencePieceText::_internal_add_pieces() { + return pieces_.Add(); } inline ::sentencepiece::SentencePieceText_SentencePiece* SentencePieceText::add_pieces() { // @@protoc_insertion_point(field_add:sentencepiece.SentencePieceText.pieces) - return pieces_.Add(); + return _internal_add_pieces(); } -inline const ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText_SentencePiece >& SentencePieceText::pieces() const { // @@protoc_insertion_point(field_list:sentencepiece.SentencePieceText.pieces) return pieces_; } // optional float score = 3; -inline bool SentencePieceText::has_score() const { - return (_has_bits_[0] & 0x00000002u) != 0; -} -inline void SentencePieceText::set_has_score() { - _has_bits_[0] |= 0x00000002u; +inline bool SentencePieceText::_internal_has_score() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + return value; } -inline void SentencePieceText::clear_has_score() { - _has_bits_[0] &= ~0x00000002u; +inline bool SentencePieceText::has_score() const { + return _internal_has_score(); } inline void SentencePieceText::clear_score() { score_ = 0; - clear_has_score(); + _has_bits_[0] &= ~0x00000002u; +} +inline float SentencePieceText::_internal_score() const { + return score_; } inline float SentencePieceText::score() const { // @@protoc_insertion_point(field_get:sentencepiece.SentencePieceText.score) - return score_; + return _internal_score(); } -inline void SentencePieceText::set_score(float value) { - set_has_score(); +inline void SentencePieceText::_internal_set_score(float value) { + _has_bits_[0] |= 0x00000002u; score_ = value; +} +inline void SentencePieceText::set_score(float value) { + _internal_set_score(value); // @@protoc_insertion_point(field_set:sentencepiece.SentencePieceText.score) } @@ -841,9 +1076,12 @@ inline void SentencePieceText::set_score(float value) { // NBestSentencePieceText // repeated .sentencepiece.SentencePieceText nbests = 1; -inline int NBestSentencePieceText::nbests_size() const { +inline int NBestSentencePieceText::_internal_nbests_size() const { return nbests_.size(); } +inline int NBestSentencePieceText::nbests_size() const { + return _internal_nbests_size(); +} inline void NBestSentencePieceText::clear_nbests() { nbests_.Clear(); } @@ -851,20 +1089,26 @@ inline ::sentencepiece::SentencePieceText* NBestSentencePieceText::mutable_nbest // @@protoc_insertion_point(field_mutable:sentencepiece.NBestSentencePieceText.nbests) return nbests_.Mutable(index); } -inline ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText >* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText >* NBestSentencePieceText::mutable_nbests() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.NBestSentencePieceText.nbests) return &nbests_; } +inline const ::sentencepiece::SentencePieceText& NBestSentencePieceText::_internal_nbests(int index) const { + return nbests_.Get(index); +} inline const ::sentencepiece::SentencePieceText& NBestSentencePieceText::nbests(int index) const { // @@protoc_insertion_point(field_get:sentencepiece.NBestSentencePieceText.nbests) - return nbests_.Get(index); + return _internal_nbests(index); +} +inline ::sentencepiece::SentencePieceText* NBestSentencePieceText::_internal_add_nbests() { + return nbests_.Add(); } inline ::sentencepiece::SentencePieceText* NBestSentencePieceText::add_nbests() { // @@protoc_insertion_point(field_add:sentencepiece.NBestSentencePieceText.nbests) - return nbests_.Add(); + return _internal_add_nbests(); } -inline const ::google::protobuf::RepeatedPtrField< ::sentencepiece::SentencePieceText >& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SentencePieceText >& NBestSentencePieceText::nbests() const { // @@protoc_insertion_point(field_list:sentencepiece.NBestSentencePieceText.nbests) return nbests_; @@ -884,4 +1128,5 @@ NBestSentencePieceText::nbests() const { // @@protoc_insertion_point(global_scope) -#endif // PROTOBUF_INCLUDED_sentencepiece_2eproto +#include +#endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_sentencepiece_2eproto diff --git a/src/builtin_pb/sentencepiece_model.pb.cc b/src/builtin_pb/sentencepiece_model.pb.cc index 65ac2fcf..59f6b1de 100644 --- a/src/builtin_pb/sentencepiece_model.pb.cc +++ b/src/builtin_pb/sentencepiece_model.pb.cc @@ -5,176 +5,152 @@ #include -#include -#include #include -#include +#include +#include #include -// This is a temporary google only hack -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS -#include "third_party/protobuf/version.h" -#endif // @@protoc_insertion_point(includes) - -namespace protobuf_sentencepiece_5fmodel_2eproto { -extern PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_5fmodel_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_ModelProto_SentencePiece; -extern PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_5fmodel_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_NormalizerSpec; -extern PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_5fmodel_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_SelfTestData_Sample; -extern PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_5fmodel_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_TrainerSpec; -extern PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_5fmodel_2eproto ::google::protobuf::internal::SCCInfo<1> scc_info_SelfTestData; -} // namespace protobuf_sentencepiece_5fmodel_2eproto +#include +extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SelfTestData_sentencepiece_5fmodel_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto; namespace sentencepiece { class TrainerSpecDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _TrainerSpec_default_instance_; class NormalizerSpecDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _NormalizerSpec_default_instance_; class SelfTestData_SampleDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _SelfTestData_Sample_default_instance_; class SelfTestDataDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _SelfTestData_default_instance_; class ModelProto_SentencePieceDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _ModelProto_SentencePiece_default_instance_; class ModelProtoDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _ModelProto_default_instance_; } // namespace sentencepiece -namespace protobuf_sentencepiece_5fmodel_2eproto { -static void InitDefaultsTrainerSpec() { +static void InitDefaultsscc_info_ModelProto_sentencepiece_5fmodel_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.DefaultConstruct(); - *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get_mutable() = ::std::string("", 5); - ::google::protobuf::internal::OnShutdownDestroyString( - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get_mutable()); - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.DefaultConstruct(); - *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get_mutable() = ::std::string("", 3); - ::google::protobuf::internal::OnShutdownDestroyString( - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get_mutable()); - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.DefaultConstruct(); - *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get_mutable() = ::std::string("", 4); - ::google::protobuf::internal::OnShutdownDestroyString( - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get_mutable()); - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.DefaultConstruct(); - *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get_mutable() = ::std::string("", 5); - ::google::protobuf::internal::OnShutdownDestroyString( - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get_mutable()); - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.DefaultConstruct(); - *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get_mutable() = ::std::string(" \342\201\207 ", 5); - ::google::protobuf::internal::OnShutdownDestroyString( - ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get_mutable()); { - void* ptr = &::sentencepiece::_TrainerSpec_default_instance_; - new (ptr) ::sentencepiece::TrainerSpec(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + void* ptr = &::sentencepiece::_ModelProto_default_instance_; + new (ptr) ::sentencepiece::ModelProto(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::sentencepiece::TrainerSpec::InitAsDefaultInstance(); + ::sentencepiece::ModelProto::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<0> scc_info_TrainerSpec = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsTrainerSpec}, {}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<4> scc_info_ModelProto_sentencepiece_5fmodel_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 4, 0, InitDefaultsscc_info_ModelProto_sentencepiece_5fmodel_2eproto}, { + &scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto.base, + &scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto.base, + &scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto.base, + &scc_info_SelfTestData_sentencepiece_5fmodel_2eproto.base,}}; -static void InitDefaultsNormalizerSpec() { +static void InitDefaultsscc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { - void* ptr = &::sentencepiece::_NormalizerSpec_default_instance_; - new (ptr) ::sentencepiece::NormalizerSpec(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + void* ptr = &::sentencepiece::_ModelProto_SentencePiece_default_instance_; + new (ptr) ::sentencepiece::ModelProto_SentencePiece(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::sentencepiece::NormalizerSpec::InitAsDefaultInstance(); + ::sentencepiece::ModelProto_SentencePiece::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<0> scc_info_NormalizerSpec = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsNormalizerSpec}, {}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto}, {}}; -static void InitDefaultsSelfTestData_Sample() { +static void InitDefaultsscc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { - void* ptr = &::sentencepiece::_SelfTestData_Sample_default_instance_; - new (ptr) ::sentencepiece::SelfTestData_Sample(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + void* ptr = &::sentencepiece::_NormalizerSpec_default_instance_; + new (ptr) ::sentencepiece::NormalizerSpec(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::sentencepiece::SelfTestData_Sample::InitAsDefaultInstance(); + ::sentencepiece::NormalizerSpec::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<0> scc_info_SelfTestData_Sample = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsSelfTestData_Sample}, {}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto}, {}}; -static void InitDefaultsSelfTestData() { +static void InitDefaultsscc_info_SelfTestData_sentencepiece_5fmodel_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { void* ptr = &::sentencepiece::_SelfTestData_default_instance_; new (ptr) ::sentencepiece::SelfTestData(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } ::sentencepiece::SelfTestData::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<1> scc_info_SelfTestData = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsSelfTestData}, { - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_SelfTestData_Sample.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SelfTestData_sentencepiece_5fmodel_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, 0, InitDefaultsscc_info_SelfTestData_sentencepiece_5fmodel_2eproto}, { + &scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto.base,}}; -static void InitDefaultsModelProto_SentencePiece() { +static void InitDefaultsscc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { - void* ptr = &::sentencepiece::_ModelProto_SentencePiece_default_instance_; - new (ptr) ::sentencepiece::ModelProto_SentencePiece(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + void* ptr = &::sentencepiece::_SelfTestData_Sample_default_instance_; + new (ptr) ::sentencepiece::SelfTestData_Sample(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::sentencepiece::ModelProto_SentencePiece::InitAsDefaultInstance(); + ::sentencepiece::SelfTestData_Sample::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<0> scc_info_ModelProto_SentencePiece = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsModelProto_SentencePiece}, {}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto}, {}}; -static void InitDefaultsModelProto() { +static void InitDefaultsscc_info_TrainerSpec_sentencepiece_5fmodel_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.DefaultConstruct(); + *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get_mutable() = std::string("", 5); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyString( + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get_mutable()); + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.DefaultConstruct(); + *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get_mutable() = std::string("", 3); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyString( + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get_mutable()); + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.DefaultConstruct(); + *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get_mutable() = std::string("", 4); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyString( + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get_mutable()); + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.DefaultConstruct(); + *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get_mutable() = std::string("", 5); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyString( + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get_mutable()); + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.DefaultConstruct(); + *::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get_mutable() = std::string(" \342\201\207 ", 5); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyString( + ::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get_mutable()); { - void* ptr = &::sentencepiece::_ModelProto_default_instance_; - new (ptr) ::sentencepiece::ModelProto(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); + void* ptr = &::sentencepiece::_TrainerSpec_default_instance_; + new (ptr) ::sentencepiece::TrainerSpec(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::sentencepiece::ModelProto::InitAsDefaultInstance(); + ::sentencepiece::TrainerSpec::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<4> scc_info_ModelProto = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 4, InitDefaultsModelProto}, { - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_ModelProto_SentencePiece.base, - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_TrainerSpec.base, - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_NormalizerSpec.base, - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_SelfTestData.base,}}; - -void InitDefaults() { - ::google::protobuf::internal::InitSCC(&scc_info_TrainerSpec.base); - ::google::protobuf::internal::InitSCC(&scc_info_NormalizerSpec.base); - ::google::protobuf::internal::InitSCC(&scc_info_SelfTestData_Sample.base); - ::google::protobuf::internal::InitSCC(&scc_info_SelfTestData.base); - ::google::protobuf::internal::InitSCC(&scc_info_ModelProto_SentencePiece.base); - ::google::protobuf::internal::InitSCC(&scc_info_ModelProto.base); -} +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_TrainerSpec_sentencepiece_5fmodel_2eproto}, {}}; -} // namespace protobuf_sentencepiece_5fmodel_2eproto namespace sentencepiece { bool TrainerSpec_ModelType_IsValid(int value) { switch (value) { @@ -188,15 +164,62 @@ bool TrainerSpec_ModelType_IsValid(int value) { } } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const TrainerSpec_ModelType TrainerSpec::UNIGRAM; -const TrainerSpec_ModelType TrainerSpec::BPE; -const TrainerSpec_ModelType TrainerSpec::WORD; -const TrainerSpec_ModelType TrainerSpec::CHAR; -const TrainerSpec_ModelType TrainerSpec::ModelType_MIN; -const TrainerSpec_ModelType TrainerSpec::ModelType_MAX; -const int TrainerSpec::ModelType_ARRAYSIZE; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 +static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed TrainerSpec_ModelType_strings[4] = {}; + +static const char TrainerSpec_ModelType_names[] = + "BPE" + "CHAR" + "UNIGRAM" + "WORD"; + +static const ::PROTOBUF_NAMESPACE_ID::internal::EnumEntry TrainerSpec_ModelType_entries[] = { + { {TrainerSpec_ModelType_names + 0, 3}, 2 }, + { {TrainerSpec_ModelType_names + 3, 4}, 4 }, + { {TrainerSpec_ModelType_names + 7, 7}, 1 }, + { {TrainerSpec_ModelType_names + 14, 4}, 3 }, +}; + +static const int TrainerSpec_ModelType_entries_by_number[] = { + 2, // 1 -> UNIGRAM + 0, // 2 -> BPE + 3, // 3 -> WORD + 1, // 4 -> CHAR +}; + +const std::string& TrainerSpec_ModelType_Name( + TrainerSpec_ModelType value) { + static const bool dummy = + ::PROTOBUF_NAMESPACE_ID::internal::InitializeEnumStrings( + TrainerSpec_ModelType_entries, + TrainerSpec_ModelType_entries_by_number, + 4, TrainerSpec_ModelType_strings); + (void) dummy; + int idx = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumName( + TrainerSpec_ModelType_entries, + TrainerSpec_ModelType_entries_by_number, + 4, value); + return idx == -1 ? ::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString() : + TrainerSpec_ModelType_strings[idx].get(); +} +bool TrainerSpec_ModelType_Parse( + const std::string& name, TrainerSpec_ModelType* value) { + int int_value; + bool success = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumValue( + TrainerSpec_ModelType_entries, 4, name, &int_value); + if (success) { + *value = static_cast(int_value); + } + return success; +} +#if (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900) +constexpr TrainerSpec_ModelType TrainerSpec::UNIGRAM; +constexpr TrainerSpec_ModelType TrainerSpec::BPE; +constexpr TrainerSpec_ModelType TrainerSpec::WORD; +constexpr TrainerSpec_ModelType TrainerSpec::CHAR; +constexpr TrainerSpec_ModelType TrainerSpec::ModelType_MIN; +constexpr TrainerSpec_ModelType TrainerSpec::ModelType_MAX; +constexpr int TrainerSpec::ModelType_ARRAYSIZE; +#endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900) bool ModelProto_SentencePiece_Type_IsValid(int value) { switch (value) { case 1: @@ -211,118 +234,252 @@ bool ModelProto_SentencePiece_Type_IsValid(int value) { } } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::NORMAL; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::UNKNOWN; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::CONTROL; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::USER_DEFINED; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::BYTE; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::UNUSED; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::Type_MIN; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece::Type_MAX; -const int ModelProto_SentencePiece::Type_ARRAYSIZE; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 +static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed ModelProto_SentencePiece_Type_strings[6] = {}; + +static const char ModelProto_SentencePiece_Type_names[] = + "BYTE" + "CONTROL" + "NORMAL" + "UNKNOWN" + "UNUSED" + "USER_DEFINED"; + +static const ::PROTOBUF_NAMESPACE_ID::internal::EnumEntry ModelProto_SentencePiece_Type_entries[] = { + { {ModelProto_SentencePiece_Type_names + 0, 4}, 6 }, + { {ModelProto_SentencePiece_Type_names + 4, 7}, 3 }, + { {ModelProto_SentencePiece_Type_names + 11, 6}, 1 }, + { {ModelProto_SentencePiece_Type_names + 17, 7}, 2 }, + { {ModelProto_SentencePiece_Type_names + 24, 6}, 5 }, + { {ModelProto_SentencePiece_Type_names + 30, 12}, 4 }, +}; + +static const int ModelProto_SentencePiece_Type_entries_by_number[] = { + 2, // 1 -> NORMAL + 3, // 2 -> UNKNOWN + 1, // 3 -> CONTROL + 5, // 4 -> USER_DEFINED + 4, // 5 -> UNUSED + 0, // 6 -> BYTE +}; + +const std::string& ModelProto_SentencePiece_Type_Name( + ModelProto_SentencePiece_Type value) { + static const bool dummy = + ::PROTOBUF_NAMESPACE_ID::internal::InitializeEnumStrings( + ModelProto_SentencePiece_Type_entries, + ModelProto_SentencePiece_Type_entries_by_number, + 6, ModelProto_SentencePiece_Type_strings); + (void) dummy; + int idx = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumName( + ModelProto_SentencePiece_Type_entries, + ModelProto_SentencePiece_Type_entries_by_number, + 6, value); + return idx == -1 ? ::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString() : + ModelProto_SentencePiece_Type_strings[idx].get(); +} +bool ModelProto_SentencePiece_Type_Parse( + const std::string& name, ModelProto_SentencePiece_Type* value) { + int int_value; + bool success = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumValue( + ModelProto_SentencePiece_Type_entries, 6, name, &int_value); + if (success) { + *value = static_cast(int_value); + } + return success; +} +#if (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900) +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::NORMAL; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::UNKNOWN; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::CONTROL; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::USER_DEFINED; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::BYTE; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::UNUSED; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::Type_MIN; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::Type_MAX; +constexpr int ModelProto_SentencePiece::Type_ARRAYSIZE; +#endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900) // =================================================================== void TrainerSpec::InitAsDefaultInstance() { } -::google::protobuf::internal::ExplicitlyConstructed<::std::string> TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_; -::google::protobuf::internal::ExplicitlyConstructed<::std::string> TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_; -::google::protobuf::internal::ExplicitlyConstructed<::std::string> TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_; -::google::protobuf::internal::ExplicitlyConstructed<::std::string> TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_; -::google::protobuf::internal::ExplicitlyConstructed<::std::string> TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_; -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int TrainerSpec::kInputFieldNumber; -const int TrainerSpec::kInputFormatFieldNumber; -const int TrainerSpec::kModelPrefixFieldNumber; -const int TrainerSpec::kModelTypeFieldNumber; -const int TrainerSpec::kVocabSizeFieldNumber; -const int TrainerSpec::kAcceptLanguageFieldNumber; -const int TrainerSpec::kSelfTestSampleSizeFieldNumber; -const int TrainerSpec::kCharacterCoverageFieldNumber; -const int TrainerSpec::kInputSentenceSizeFieldNumber; -const int TrainerSpec::kShuffleInputSentenceFieldNumber; -const int TrainerSpec::kMiningSentenceSizeFieldNumber; -const int TrainerSpec::kTrainingSentenceSizeFieldNumber; -const int TrainerSpec::kSeedSentencepieceSizeFieldNumber; -const int TrainerSpec::kShrinkingFactorFieldNumber; -const int TrainerSpec::kMaxSentenceLengthFieldNumber; -const int TrainerSpec::kNumThreadsFieldNumber; -const int TrainerSpec::kNumSubIterationsFieldNumber; -const int TrainerSpec::kMaxSentencepieceLengthFieldNumber; -const int TrainerSpec::kSplitByUnicodeScriptFieldNumber; -const int TrainerSpec::kSplitByNumberFieldNumber; -const int TrainerSpec::kSplitByWhitespaceFieldNumber; -const int TrainerSpec::kTreatWhitespaceAsSuffixFieldNumber; -const int TrainerSpec::kSplitDigitsFieldNumber; -const int TrainerSpec::kControlSymbolsFieldNumber; -const int TrainerSpec::kUserDefinedSymbolsFieldNumber; -const int TrainerSpec::kRequiredCharsFieldNumber; -const int TrainerSpec::kByteFallbackFieldNumber; -const int TrainerSpec::kVocabularyOutputPieceScoreFieldNumber; -const int TrainerSpec::kHardVocabLimitFieldNumber; -const int TrainerSpec::kUseAllVocabFieldNumber; -const int TrainerSpec::kUnkIdFieldNumber; -const int TrainerSpec::kBosIdFieldNumber; -const int TrainerSpec::kEosIdFieldNumber; -const int TrainerSpec::kPadIdFieldNumber; -const int TrainerSpec::kUnkPieceFieldNumber; -const int TrainerSpec::kBosPieceFieldNumber; -const int TrainerSpec::kEosPieceFieldNumber; -const int TrainerSpec::kPadPieceFieldNumber; -const int TrainerSpec::kUnkSurfaceFieldNumber; -const int TrainerSpec::kTrainExtremelyLargeCorpusFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -TrainerSpec::TrainerSpec() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_TrainerSpec.base); +class TrainerSpec::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static void set_has_input_format(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } + static void set_has_model_prefix(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static void set_has_model_type(HasBits* has_bits) { + (*has_bits)[0] |= 262144u; + } + static void set_has_vocab_size(HasBits* has_bits) { + (*has_bits)[0] |= 524288u; + } + static void set_has_self_test_sample_size(HasBits* has_bits) { + (*has_bits)[0] |= 256u; + } + static void set_has_character_coverage(HasBits* has_bits) { + (*has_bits)[0] |= 1048576u; + } + static void set_has_input_sentence_size(HasBits* has_bits) { + (*has_bits)[0] |= 512u; + } + static void set_has_shuffle_input_sentence(HasBits* has_bits) { + (*has_bits)[0] |= 134217728u; + } + static void set_has_mining_sentence_size(HasBits* has_bits) { + (*has_bits)[0] |= 1024u; + } + static void set_has_training_sentence_size(HasBits* has_bits) { + (*has_bits)[0] |= 2048u; + } + static void set_has_seed_sentencepiece_size(HasBits* has_bits) { + (*has_bits)[0] |= 2097152u; + } + static void set_has_shrinking_factor(HasBits* has_bits) { + (*has_bits)[0] |= 4194304u; + } + static void set_has_max_sentence_length(HasBits* has_bits) { + (*has_bits)[0] |= 33554432u; + } + static void set_has_num_threads(HasBits* has_bits) { + (*has_bits)[0] |= 8388608u; + } + static void set_has_num_sub_iterations(HasBits* has_bits) { + (*has_bits)[0] |= 16777216u; + } + static void set_has_max_sentencepiece_length(HasBits* has_bits) { + (*has_bits)[0] |= 67108864u; + } + static void set_has_split_by_unicode_script(HasBits* has_bits) { + (*has_bits)[0] |= 268435456u; + } + static void set_has_split_by_number(HasBits* has_bits) { + (*has_bits)[0] |= 536870912u; + } + static void set_has_split_by_whitespace(HasBits* has_bits) { + (*has_bits)[0] |= 1073741824u; + } + static void set_has_treat_whitespace_as_suffix(HasBits* has_bits) { + (*has_bits)[0] |= 4096u; + } + static void set_has_split_digits(HasBits* has_bits) { + (*has_bits)[0] |= 8192u; + } + static void set_has_required_chars(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } + static void set_has_byte_fallback(HasBits* has_bits) { + (*has_bits)[0] |= 16384u; + } + static void set_has_vocabulary_output_piece_score(HasBits* has_bits) { + (*has_bits)[0] |= 2147483648u; + } + static void set_has_hard_vocab_limit(HasBits* has_bits) { + (*has_bits)[1] |= 1u; + } + static void set_has_use_all_vocab(HasBits* has_bits) { + (*has_bits)[0] |= 32768u; + } + static void set_has_unk_id(HasBits* has_bits) { + (*has_bits)[0] |= 65536u; + } + static void set_has_bos_id(HasBits* has_bits) { + (*has_bits)[1] |= 2u; + } + static void set_has_eos_id(HasBits* has_bits) { + (*has_bits)[1] |= 4u; + } + static void set_has_pad_id(HasBits* has_bits) { + (*has_bits)[1] |= 8u; + } + static void set_has_unk_piece(HasBits* has_bits) { + (*has_bits)[0] |= 16u; + } + static void set_has_bos_piece(HasBits* has_bits) { + (*has_bits)[0] |= 32u; + } + static void set_has_eos_piece(HasBits* has_bits) { + (*has_bits)[0] |= 64u; + } + static void set_has_pad_piece(HasBits* has_bits) { + (*has_bits)[0] |= 128u; + } + static void set_has_unk_surface(HasBits* has_bits) { + (*has_bits)[0] |= 8u; + } + static void set_has_train_extremely_large_corpus(HasBits* has_bits) { + (*has_bits)[0] |= 131072u; + } +}; + +::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_; +::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_; +::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_; +::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_; +::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_; +TrainerSpec::TrainerSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + _extensions_(arena), + input_(arena), + accept_language_(arena), + control_symbols_(arena), + user_defined_symbols_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.TrainerSpec) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.TrainerSpec) } TrainerSpec::TrainerSpec(const TrainerSpec& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_), input_(from.input_), accept_language_(from.accept_language_), control_symbols_(from.control_symbols_), user_defined_symbols_(from.user_defined_symbols_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); _extensions_.MergeFrom(from._extensions_); - model_prefix_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_model_prefix()) { - model_prefix_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.model_prefix_); + model_prefix_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_model_prefix()) { + model_prefix_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_model_prefix(), + GetArena()); } - input_format_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_input_format()) { - input_format_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.input_format_); + input_format_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_input_format()) { + input_format_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_input_format(), + GetArena()); } - required_chars_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_required_chars()) { - required_chars_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.required_chars_); + required_chars_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_required_chars()) { + required_chars_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_required_chars(), + GetArena()); } unk_surface_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get()); - if (from.has_unk_surface()) { - unk_surface_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), from.unk_surface_); + if (from._internal_has_unk_surface()) { + unk_surface_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), from._internal_unk_surface(), + GetArena()); } unk_piece_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get()); - if (from.has_unk_piece()) { - unk_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), from.unk_piece_); + if (from._internal_has_unk_piece()) { + unk_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), from._internal_unk_piece(), + GetArena()); } bos_piece_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get()); - if (from.has_bos_piece()) { - bos_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), from.bos_piece_); + if (from._internal_has_bos_piece()) { + bos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), from._internal_bos_piece(), + GetArena()); } eos_piece_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get()); - if (from.has_eos_piece()) { - eos_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), from.eos_piece_); + if (from._internal_has_eos_piece()) { + eos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), from._internal_eos_piece(), + GetArena()); } pad_piece_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get()); - if (from.has_pad_piece()) { - pad_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), from.pad_piece_); + if (from._internal_has_pad_piece()) { + pad_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), from._internal_pad_piece(), + GetArena()); } ::memcpy(&self_test_sample_size_, &from.self_test_sample_size_, static_cast(reinterpret_cast(&pad_id_) - @@ -331,9 +488,10 @@ TrainerSpec::TrainerSpec(const TrainerSpec& from) } void TrainerSpec::SharedCtor() { - model_prefix_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - input_format_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - required_chars_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto.base); + model_prefix_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + input_format_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + required_chars_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); unk_surface_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get()); unk_piece_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get()); bos_piece_.UnsafeSetDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get()); @@ -365,12 +523,14 @@ void TrainerSpec::SharedCtor() { TrainerSpec::~TrainerSpec() { // @@protoc_insertion_point(destructor:sentencepiece.TrainerSpec) SharedDtor(); + _internal_metadata_.Delete(); } void TrainerSpec::SharedDtor() { - model_prefix_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - input_format_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - required_chars_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + GOOGLE_DCHECK(GetArena() == nullptr); + model_prefix_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + input_format_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + required_chars_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); unk_surface_.DestroyNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get()); unk_piece_.DestroyNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get()); bos_piece_.DestroyNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get()); @@ -378,18 +538,24 @@ void TrainerSpec::SharedDtor() { pad_piece_.DestroyNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get()); } +void TrainerSpec::ArenaDtor(void* object) { + TrainerSpec* _this = reinterpret_cast< TrainerSpec* >(object); + (void)_this; +} +void TrainerSpec::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void TrainerSpec::SetCachedSize(int size) const { _cached_size_.Set(size); } const TrainerSpec& TrainerSpec::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_5fmodel_2eproto::scc_info_TrainerSpec.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto.base); return *internal_default_instance(); } void TrainerSpec::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.TrainerSpec) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; @@ -399,38 +565,38 @@ void TrainerSpec::Clear() { control_symbols_.Clear(); user_defined_symbols_.Clear(); cached_has_bits = _has_bits_[0]; - if (cached_has_bits & 255u) { + if (cached_has_bits & 0x000000ffu) { if (cached_has_bits & 0x00000001u) { - model_prefix_.ClearNonDefaultToEmptyNoArena(); + model_prefix_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000002u) { - input_format_.ClearNonDefaultToEmptyNoArena(); + input_format_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000004u) { - required_chars_.ClearNonDefaultToEmptyNoArena(); + required_chars_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000008u) { - unk_surface_.UnsafeMutablePointer()->assign(*&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get()); + unk_surface_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), GetArena()); } if (cached_has_bits & 0x00000010u) { - unk_piece_.UnsafeMutablePointer()->assign(*&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get()); + unk_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), GetArena()); } if (cached_has_bits & 0x00000020u) { - bos_piece_.UnsafeMutablePointer()->assign(*&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get()); + bos_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), GetArena()); } if (cached_has_bits & 0x00000040u) { - eos_piece_.UnsafeMutablePointer()->assign(*&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get()); + eos_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), GetArena()); } if (cached_has_bits & 0x00000080u) { - pad_piece_.UnsafeMutablePointer()->assign(*&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get()); + pad_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), GetArena()); } } - if (cached_has_bits & 65280u) { + if (cached_has_bits & 0x0000ff00u) { ::memset(&self_test_sample_size_, 0, static_cast( reinterpret_cast(&use_all_vocab_) - reinterpret_cast(&self_test_sample_size_)) + sizeof(use_all_vocab_)); } - if (cached_has_bits & 16711680u) { + if (cached_has_bits & 0x00ff0000u) { ::memset(&unk_id_, 0, static_cast( reinterpret_cast(&train_extremely_large_corpus_) - reinterpret_cast(&unk_id_)) + sizeof(train_extremely_large_corpus_)); @@ -441,7 +607,7 @@ void TrainerSpec::Clear() { shrinking_factor_ = 0.75f; num_threads_ = 16; } - if (cached_has_bits & 4278190080u) { + if (cached_has_bits & 0xff000000u) { num_sub_iterations_ = 2; max_sentence_length_ = 4192; max_sentencepiece_length_ = 16; @@ -452,832 +618,658 @@ void TrainerSpec::Clear() { vocabulary_output_piece_score_ = true; } cached_has_bits = _has_bits_[1]; - if (cached_has_bits & 15u) { + if (cached_has_bits & 0x0000000fu) { hard_vocab_limit_ = true; bos_id_ = 1; eos_id_ = 2; pad_id_ = -1; } _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool TrainerSpec::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.TrainerSpec) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(16383u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // repeated string input = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->add_input())); - } else { - goto handle_unusual; - } - break; - } + _internal_metadata_.Clear(); +} +const char* TrainerSpec::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // repeated string input = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + auto str = _internal_add_input(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); + } else goto handle_unusual; + continue; // optional string model_prefix = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_model_prefix())); - } else { - goto handle_unusual; - } - break; - } - + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + auto str = _internal_mutable_model_prefix(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default = UNIGRAM]; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(24u /* 24 & 0xFF */)) { - int value; - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( - input, &value))); - if (::sentencepiece::TrainerSpec_ModelType_IsValid(value)) { - set_model_type(static_cast< ::sentencepiece::TrainerSpec_ModelType >(value)); + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + ::PROTOBUF_NAMESPACE_ID::uint64 val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + if (PROTOBUF_PREDICT_TRUE(::sentencepiece::TrainerSpec_ModelType_IsValid(val))) { + _internal_set_model_type(static_cast<::sentencepiece::TrainerSpec_ModelType>(val)); } else { - unknown_fields_stream.WriteVarint32(24u); - unknown_fields_stream.WriteVarint32( - static_cast< ::google::protobuf::uint32>(value)); + ::PROTOBUF_NAMESPACE_ID::internal::WriteVarint(3, val, mutable_unknown_fields()); } - } else { - goto handle_unusual; - } - break; - } - + } else goto handle_unusual; + continue; // optional int32 vocab_size = 4 [default = 8000]; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(32u /* 32 & 0xFF */)) { - set_has_vocab_size(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &vocab_size_))); - } else { - goto handle_unusual; - } - break; - } - + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { + _Internal::set_has_vocab_size(&_has_bits_); + vocab_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // repeated string accept_language = 5; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(42u /* 42 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->add_accept_language())); - } else { - goto handle_unusual; - } - break; - } - + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { + ptr -= 1; + do { + ptr += 1; + auto str = _internal_add_accept_language(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<42>(ptr)); + } else goto handle_unusual; + continue; // optional int32 self_test_sample_size = 6 [default = 0]; - case 6: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(48u /* 48 & 0xFF */)) { - set_has_self_test_sample_size(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &self_test_sample_size_))); - } else { - goto handle_unusual; - } - break; - } - + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 48)) { + _Internal::set_has_self_test_sample_size(&_has_bits_); + self_test_sample_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string input_format = 7; - case 7: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(58u /* 58 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_input_format())); - } else { - goto handle_unusual; - } - break; - } - + case 7: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 58)) { + auto str = _internal_mutable_input_format(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional float character_coverage = 10 [default = 0.9995]; - case 10: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(85u /* 85 & 0xFF */)) { - set_has_character_coverage(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( - input, &character_coverage_))); - } else { - goto handle_unusual; - } - break; - } - + case 10: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 85)) { + _Internal::set_has_character_coverage(&_has_bits_); + character_coverage_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(float); + } else goto handle_unusual; + continue; // optional int32 input_sentence_size = 11 [default = 0]; - case 11: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(88u /* 88 & 0xFF */)) { - set_has_input_sentence_size(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &input_sentence_size_))); - } else { - goto handle_unusual; - } - break; - } - + case 11: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 88)) { + _Internal::set_has_input_sentence_size(&_has_bits_); + input_sentence_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 mining_sentence_size = 12 [deprecated = true]; - case 12: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(96u /* 96 & 0xFF */)) { - set_has_mining_sentence_size(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &mining_sentence_size_))); - } else { - goto handle_unusual; - } - break; - } - + case 12: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 96)) { + _Internal::set_has_mining_sentence_size(&_has_bits_); + mining_sentence_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 training_sentence_size = 13 [deprecated = true]; - case 13: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(104u /* 104 & 0xFF */)) { - set_has_training_sentence_size(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &training_sentence_size_))); - } else { - goto handle_unusual; - } - break; - } - + case 13: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 104)) { + _Internal::set_has_training_sentence_size(&_has_bits_); + training_sentence_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 seed_sentencepiece_size = 14 [default = 1000000]; - case 14: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(112u /* 112 & 0xFF */)) { - set_has_seed_sentencepiece_size(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &seed_sentencepiece_size_))); - } else { - goto handle_unusual; - } - break; - } - + case 14: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 112)) { + _Internal::set_has_seed_sentencepiece_size(&_has_bits_); + seed_sentencepiece_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional float shrinking_factor = 15 [default = 0.75]; - case 15: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(125u /* 125 & 0xFF */)) { - set_has_shrinking_factor(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( - input, &shrinking_factor_))); - } else { - goto handle_unusual; - } - break; - } - + case 15: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 125)) { + _Internal::set_has_shrinking_factor(&_has_bits_); + shrinking_factor_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(float); + } else goto handle_unusual; + continue; // optional int32 num_threads = 16 [default = 16]; - case 16: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(128u /* 128 & 0xFF */)) { - set_has_num_threads(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &num_threads_))); - } else { - goto handle_unusual; - } - break; - } - + case 16: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 128)) { + _Internal::set_has_num_threads(&_has_bits_); + num_threads_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 num_sub_iterations = 17 [default = 2]; - case 17: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(136u /* 136 & 0xFF */)) { - set_has_num_sub_iterations(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &num_sub_iterations_))); - } else { - goto handle_unusual; - } - break; - } - + case 17: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 136)) { + _Internal::set_has_num_sub_iterations(&_has_bits_); + num_sub_iterations_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 max_sentence_length = 18 [default = 4192]; - case 18: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(144u /* 144 & 0xFF */)) { - set_has_max_sentence_length(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &max_sentence_length_))); - } else { - goto handle_unusual; - } - break; - } - + case 18: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 144)) { + _Internal::set_has_max_sentence_length(&_has_bits_); + max_sentence_length_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool shuffle_input_sentence = 19 [default = true]; - case 19: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(152u /* 152 & 0xFF */)) { - set_has_shuffle_input_sentence(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &shuffle_input_sentence_))); - } else { - goto handle_unusual; - } - break; - } - + case 19: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 152)) { + _Internal::set_has_shuffle_input_sentence(&_has_bits_); + shuffle_input_sentence_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 max_sentencepiece_length = 20 [default = 16]; - case 20: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(160u /* 160 & 0xFF */)) { - set_has_max_sentencepiece_length(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &max_sentencepiece_length_))); - } else { - goto handle_unusual; - } - break; - } - + case 20: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 160)) { + _Internal::set_has_max_sentencepiece_length(&_has_bits_); + max_sentencepiece_length_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool split_by_unicode_script = 21 [default = true]; - case 21: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(168u /* 168 & 0xFF */)) { - set_has_split_by_unicode_script(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &split_by_unicode_script_))); - } else { - goto handle_unusual; - } - break; - } - + case 21: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 168)) { + _Internal::set_has_split_by_unicode_script(&_has_bits_); + split_by_unicode_script_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool split_by_whitespace = 22 [default = true]; - case 22: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(176u /* 176 & 0xFF */)) { - set_has_split_by_whitespace(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &split_by_whitespace_))); - } else { - goto handle_unusual; - } - break; - } - + case 22: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 176)) { + _Internal::set_has_split_by_whitespace(&_has_bits_); + split_by_whitespace_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool split_by_number = 23 [default = true]; - case 23: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(184u /* 184 & 0xFF */)) { - set_has_split_by_number(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &split_by_number_))); - } else { - goto handle_unusual; - } - break; - } - + case 23: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 184)) { + _Internal::set_has_split_by_number(&_has_bits_); + split_by_number_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool treat_whitespace_as_suffix = 24 [default = false]; - case 24: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(192u /* 192 & 0xFF */)) { - set_has_treat_whitespace_as_suffix(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &treat_whitespace_as_suffix_))); - } else { - goto handle_unusual; - } - break; - } - + case 24: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 192)) { + _Internal::set_has_treat_whitespace_as_suffix(&_has_bits_); + treat_whitespace_as_suffix_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool split_digits = 25 [default = false]; - case 25: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(200u /* 200 & 0xFF */)) { - set_has_split_digits(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &split_digits_))); - } else { - goto handle_unusual; - } - break; - } - + case 25: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 200)) { + _Internal::set_has_split_digits(&_has_bits_); + split_digits_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // repeated string control_symbols = 30; - case 30: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(242u /* 242 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->add_control_symbols())); - } else { - goto handle_unusual; - } - break; - } - + case 30: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 242)) { + ptr -= 2; + do { + ptr += 2; + auto str = _internal_add_control_symbols(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<242>(ptr)); + } else goto handle_unusual; + continue; // repeated string user_defined_symbols = 31; - case 31: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(250u /* 250 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->add_user_defined_symbols())); - } else { - goto handle_unusual; - } - break; - } - + case 31: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 250)) { + ptr -= 2; + do { + ptr += 2; + auto str = _internal_add_user_defined_symbols(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<250>(ptr)); + } else goto handle_unusual; + continue; // optional bool vocabulary_output_piece_score = 32 [default = true]; - case 32: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(0u /* 256 & 0xFF */)) { - set_has_vocabulary_output_piece_score(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &vocabulary_output_piece_score_))); - } else { - goto handle_unusual; - } - break; - } - + case 32: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 0)) { + _Internal::set_has_vocabulary_output_piece_score(&_has_bits_); + vocabulary_output_piece_score_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool hard_vocab_limit = 33 [default = true]; - case 33: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(8u /* 264 & 0xFF */)) { - set_has_hard_vocab_limit(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &hard_vocab_limit_))); - } else { - goto handle_unusual; - } - break; - } - + case 33: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) { + _Internal::set_has_hard_vocab_limit(&_has_bits_); + hard_vocab_limit_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool use_all_vocab = 34 [default = false]; - case 34: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(16u /* 272 & 0xFF */)) { - set_has_use_all_vocab(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &use_all_vocab_))); - } else { - goto handle_unusual; - } - break; - } - + case 34: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + _Internal::set_has_use_all_vocab(&_has_bits_); + use_all_vocab_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool byte_fallback = 35 [default = false]; - case 35: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(24u /* 280 & 0xFF */)) { - set_has_byte_fallback(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &byte_fallback_))); - } else { - goto handle_unusual; - } - break; - } - + case 35: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + _Internal::set_has_byte_fallback(&_has_bits_); + byte_fallback_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string required_chars = 36; - case 36: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(34u /* 290 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_required_chars())); - } else { - goto handle_unusual; - } - break; - } - + case 36: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) { + auto str = _internal_mutable_required_chars(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 unk_id = 40 [default = 0]; - case 40: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(64u /* 320 & 0xFF */)) { - set_has_unk_id(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &unk_id_))); - } else { - goto handle_unusual; - } - break; - } - + case 40: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 64)) { + _Internal::set_has_unk_id(&_has_bits_); + unk_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 bos_id = 41 [default = 1]; - case 41: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(72u /* 328 & 0xFF */)) { - set_has_bos_id(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &bos_id_))); - } else { - goto handle_unusual; - } - break; - } - + case 41: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 72)) { + _Internal::set_has_bos_id(&_has_bits_); + bos_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 eos_id = 42 [default = 2]; - case 42: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(80u /* 336 & 0xFF */)) { - set_has_eos_id(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &eos_id_))); - } else { - goto handle_unusual; - } - break; - } - + case 42: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 80)) { + _Internal::set_has_eos_id(&_has_bits_); + eos_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional int32 pad_id = 43 [default = -1]; - case 43: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(88u /* 344 & 0xFF */)) { - set_has_pad_id(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &pad_id_))); - } else { - goto handle_unusual; - } - break; - } - + case 43: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 88)) { + _Internal::set_has_pad_id(&_has_bits_); + pad_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string unk_surface = 44 [default = " \342\201\207 "]; - case 44: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(98u /* 354 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_unk_surface())); - } else { - goto handle_unusual; - } - break; - } - + case 44: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 98)) { + auto str = _internal_mutable_unk_surface(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string unk_piece = 45 [default = ""]; - case 45: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(106u /* 362 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_unk_piece())); - } else { - goto handle_unusual; - } - break; - } - + case 45: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 106)) { + auto str = _internal_mutable_unk_piece(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string bos_piece = 46 [default = ""]; - case 46: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(114u /* 370 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_bos_piece())); - } else { - goto handle_unusual; - } - break; - } - + case 46: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 114)) { + auto str = _internal_mutable_bos_piece(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string eos_piece = 47 [default = ""]; - case 47: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(122u /* 378 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_eos_piece())); - } else { - goto handle_unusual; - } - break; - } - + case 47: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 122)) { + auto str = _internal_mutable_eos_piece(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string pad_piece = 48 [default = ""]; - case 48: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(130u /* 386 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_pad_piece())); - } else { - goto handle_unusual; - } - break; - } - + case 48: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 130)) { + auto str = _internal_mutable_pad_piece(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool train_extremely_large_corpus = 49 [default = false]; - case 49: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(136u /* 392 & 0xFF */)) { - set_has_train_extremely_large_corpus(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &train_extremely_large_corpus_))); - } else { - goto handle_unusual; - } - break; - } - + case 49: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 136)) { + _Internal::set_has_train_extremely_large_corpus(&_has_bits_); + train_extremely_large_corpus_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - if ((1600u <= tag)) { - DO_(_extensions_.ParseField(tag, input, - internal_default_instance(), - &unknown_fields_stream)); - continue; - } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + if ((1600u <= tag)) { + ptr = _extensions_.ParseField(tag, ptr, + internal_default_instance(), &_internal_metadata_, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.TrainerSpec) - return true; + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.TrainerSpec) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void TrainerSpec::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.TrainerSpec) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* TrainerSpec::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.TrainerSpec) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; // repeated string input = 1; - for (int i = 0, n = this->input_size(); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteString( - 1, this->input(i), output); + for (int i = 0, n = this->_internal_input_size(); i < n; i++) { + const auto& s = this->_internal_input(i); + target = stream->WriteString(1, s, target); } cached_has_bits = _has_bits_[0]; // optional string model_prefix = 2; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->model_prefix(), output); + target = stream->WriteStringMaybeAliased( + 2, this->_internal_model_prefix(), target); } // optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default = UNIGRAM]; if (cached_has_bits & 0x00040000u) { - ::google::protobuf::internal::WireFormatLite::WriteEnum( - 3, this->model_type(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray( + 3, this->_internal_model_type(), target); } // optional int32 vocab_size = 4 [default = 8000]; if (cached_has_bits & 0x00080000u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(4, this->vocab_size(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->_internal_vocab_size(), target); } // repeated string accept_language = 5; - for (int i = 0, n = this->accept_language_size(); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteString( - 5, this->accept_language(i), output); + for (int i = 0, n = this->_internal_accept_language_size(); i < n; i++) { + const auto& s = this->_internal_accept_language(i); + target = stream->WriteString(5, s, target); } // optional int32 self_test_sample_size = 6 [default = 0]; if (cached_has_bits & 0x00000100u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(6, this->self_test_sample_size(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(6, this->_internal_self_test_sample_size(), target); } // optional string input_format = 7; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 7, this->input_format(), output); + target = stream->WriteStringMaybeAliased( + 7, this->_internal_input_format(), target); } // optional float character_coverage = 10 [default = 0.9995]; if (cached_has_bits & 0x00100000u) { - ::google::protobuf::internal::WireFormatLite::WriteFloat(10, this->character_coverage(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(10, this->_internal_character_coverage(), target); } // optional int32 input_sentence_size = 11 [default = 0]; if (cached_has_bits & 0x00000200u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(11, this->input_sentence_size(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(11, this->_internal_input_sentence_size(), target); } // optional int32 mining_sentence_size = 12 [deprecated = true]; if (cached_has_bits & 0x00000400u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(12, this->mining_sentence_size(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(12, this->_internal_mining_sentence_size(), target); } // optional int32 training_sentence_size = 13 [deprecated = true]; if (cached_has_bits & 0x00000800u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(13, this->training_sentence_size(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(13, this->_internal_training_sentence_size(), target); } // optional int32 seed_sentencepiece_size = 14 [default = 1000000]; if (cached_has_bits & 0x00200000u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(14, this->seed_sentencepiece_size(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(14, this->_internal_seed_sentencepiece_size(), target); } // optional float shrinking_factor = 15 [default = 0.75]; if (cached_has_bits & 0x00400000u) { - ::google::protobuf::internal::WireFormatLite::WriteFloat(15, this->shrinking_factor(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(15, this->_internal_shrinking_factor(), target); } // optional int32 num_threads = 16 [default = 16]; if (cached_has_bits & 0x00800000u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(16, this->num_threads(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(16, this->_internal_num_threads(), target); } // optional int32 num_sub_iterations = 17 [default = 2]; if (cached_has_bits & 0x01000000u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(17, this->num_sub_iterations(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(17, this->_internal_num_sub_iterations(), target); } // optional int32 max_sentence_length = 18 [default = 4192]; if (cached_has_bits & 0x02000000u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(18, this->max_sentence_length(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(18, this->_internal_max_sentence_length(), target); } // optional bool shuffle_input_sentence = 19 [default = true]; if (cached_has_bits & 0x08000000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(19, this->shuffle_input_sentence(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(19, this->_internal_shuffle_input_sentence(), target); } // optional int32 max_sentencepiece_length = 20 [default = 16]; if (cached_has_bits & 0x04000000u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(20, this->max_sentencepiece_length(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(20, this->_internal_max_sentencepiece_length(), target); } // optional bool split_by_unicode_script = 21 [default = true]; if (cached_has_bits & 0x10000000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(21, this->split_by_unicode_script(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(21, this->_internal_split_by_unicode_script(), target); } // optional bool split_by_whitespace = 22 [default = true]; if (cached_has_bits & 0x40000000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(22, this->split_by_whitespace(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(22, this->_internal_split_by_whitespace(), target); } // optional bool split_by_number = 23 [default = true]; if (cached_has_bits & 0x20000000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(23, this->split_by_number(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(23, this->_internal_split_by_number(), target); } // optional bool treat_whitespace_as_suffix = 24 [default = false]; if (cached_has_bits & 0x00001000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(24, this->treat_whitespace_as_suffix(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(24, this->_internal_treat_whitespace_as_suffix(), target); } // optional bool split_digits = 25 [default = false]; if (cached_has_bits & 0x00002000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(25, this->split_digits(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(25, this->_internal_split_digits(), target); } // repeated string control_symbols = 30; - for (int i = 0, n = this->control_symbols_size(); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteString( - 30, this->control_symbols(i), output); + for (int i = 0, n = this->_internal_control_symbols_size(); i < n; i++) { + const auto& s = this->_internal_control_symbols(i); + target = stream->WriteString(30, s, target); } // repeated string user_defined_symbols = 31; - for (int i = 0, n = this->user_defined_symbols_size(); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteString( - 31, this->user_defined_symbols(i), output); + for (int i = 0, n = this->_internal_user_defined_symbols_size(); i < n; i++) { + const auto& s = this->_internal_user_defined_symbols(i); + target = stream->WriteString(31, s, target); } // optional bool vocabulary_output_piece_score = 32 [default = true]; if (cached_has_bits & 0x80000000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(32, this->vocabulary_output_piece_score(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(32, this->_internal_vocabulary_output_piece_score(), target); } cached_has_bits = _has_bits_[1]; // optional bool hard_vocab_limit = 33 [default = true]; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(33, this->hard_vocab_limit(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(33, this->_internal_hard_vocab_limit(), target); } cached_has_bits = _has_bits_[0]; // optional bool use_all_vocab = 34 [default = false]; if (cached_has_bits & 0x00008000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(34, this->use_all_vocab(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(34, this->_internal_use_all_vocab(), target); } // optional bool byte_fallback = 35 [default = false]; if (cached_has_bits & 0x00004000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(35, this->byte_fallback(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(35, this->_internal_byte_fallback(), target); } // optional string required_chars = 36; if (cached_has_bits & 0x00000004u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 36, this->required_chars(), output); + target = stream->WriteStringMaybeAliased( + 36, this->_internal_required_chars(), target); } // optional int32 unk_id = 40 [default = 0]; if (cached_has_bits & 0x00010000u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(40, this->unk_id(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(40, this->_internal_unk_id(), target); } cached_has_bits = _has_bits_[1]; // optional int32 bos_id = 41 [default = 1]; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(41, this->bos_id(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(41, this->_internal_bos_id(), target); } // optional int32 eos_id = 42 [default = 2]; if (cached_has_bits & 0x00000004u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(42, this->eos_id(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(42, this->_internal_eos_id(), target); } // optional int32 pad_id = 43 [default = -1]; if (cached_has_bits & 0x00000008u) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(43, this->pad_id(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(43, this->_internal_pad_id(), target); } cached_has_bits = _has_bits_[0]; // optional string unk_surface = 44 [default = " \342\201\207 "]; if (cached_has_bits & 0x00000008u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 44, this->unk_surface(), output); + target = stream->WriteStringMaybeAliased( + 44, this->_internal_unk_surface(), target); } // optional string unk_piece = 45 [default = ""]; if (cached_has_bits & 0x00000010u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 45, this->unk_piece(), output); + target = stream->WriteStringMaybeAliased( + 45, this->_internal_unk_piece(), target); } // optional string bos_piece = 46 [default = ""]; if (cached_has_bits & 0x00000020u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 46, this->bos_piece(), output); + target = stream->WriteStringMaybeAliased( + 46, this->_internal_bos_piece(), target); } // optional string eos_piece = 47 [default = ""]; if (cached_has_bits & 0x00000040u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 47, this->eos_piece(), output); + target = stream->WriteStringMaybeAliased( + 47, this->_internal_eos_piece(), target); } // optional string pad_piece = 48 [default = ""]; if (cached_has_bits & 0x00000080u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 48, this->pad_piece(), output); + target = stream->WriteStringMaybeAliased( + 48, this->_internal_pad_piece(), target); } // optional bool train_extremely_large_corpus = 49 [default = false]; if (cached_has_bits & 0x00020000u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(49, this->train_extremely_large_corpus(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(49, this->_internal_train_extremely_large_corpus(), target); } // Extension range [200, 536870912) - _extensions_.SerializeWithCachedSizes( - 200, 536870912, output); + target = _extensions_._InternalSerialize( + 200, 536870912, target, stream); - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.TrainerSpec) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.TrainerSpec) + return target; } size_t TrainerSpec::ByteSizeLong() const { @@ -1286,291 +1278,299 @@ size_t TrainerSpec::ByteSizeLong() const { total_size += _extensions_.ByteSize(); - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; // repeated string input = 1; total_size += 1 * - ::google::protobuf::internal::FromIntSize(this->input_size()); - for (int i = 0, n = this->input_size(); i < n; i++) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - this->input(i)); + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(input_.size()); + for (int i = 0, n = input_.size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + input_.Get(i)); } // repeated string accept_language = 5; total_size += 1 * - ::google::protobuf::internal::FromIntSize(this->accept_language_size()); - for (int i = 0, n = this->accept_language_size(); i < n; i++) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - this->accept_language(i)); + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(accept_language_.size()); + for (int i = 0, n = accept_language_.size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + accept_language_.Get(i)); } // repeated string control_symbols = 30; total_size += 2 * - ::google::protobuf::internal::FromIntSize(this->control_symbols_size()); - for (int i = 0, n = this->control_symbols_size(); i < n; i++) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - this->control_symbols(i)); + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(control_symbols_.size()); + for (int i = 0, n = control_symbols_.size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + control_symbols_.Get(i)); } // repeated string user_defined_symbols = 31; total_size += 2 * - ::google::protobuf::internal::FromIntSize(this->user_defined_symbols_size()); - for (int i = 0, n = this->user_defined_symbols_size(); i < n; i++) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - this->user_defined_symbols(i)); + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(user_defined_symbols_.size()); + for (int i = 0, n = user_defined_symbols_.size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + user_defined_symbols_.Get(i)); } - if (_has_bits_[0 / 32] & 255u) { + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x000000ffu) { // optional string model_prefix = 2; - if (has_model_prefix()) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->model_prefix()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_model_prefix()); } // optional string input_format = 7; - if (has_input_format()) { + if (cached_has_bits & 0x00000002u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->input_format()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_input_format()); } // optional string required_chars = 36; - if (has_required_chars()) { + if (cached_has_bits & 0x00000004u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->required_chars()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_required_chars()); } // optional string unk_surface = 44 [default = " \342\201\207 "]; - if (has_unk_surface()) { + if (cached_has_bits & 0x00000008u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->unk_surface()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_unk_surface()); } // optional string unk_piece = 45 [default = ""]; - if (has_unk_piece()) { + if (cached_has_bits & 0x00000010u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->unk_piece()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_unk_piece()); } // optional string bos_piece = 46 [default = ""]; - if (has_bos_piece()) { + if (cached_has_bits & 0x00000020u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->bos_piece()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_bos_piece()); } // optional string eos_piece = 47 [default = ""]; - if (has_eos_piece()) { + if (cached_has_bits & 0x00000040u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->eos_piece()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_eos_piece()); } // optional string pad_piece = 48 [default = ""]; - if (has_pad_piece()) { + if (cached_has_bits & 0x00000080u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->pad_piece()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_pad_piece()); } } - if (_has_bits_[8 / 32] & 65280u) { + if (cached_has_bits & 0x0000ff00u) { // optional int32 self_test_sample_size = 6 [default = 0]; - if (has_self_test_sample_size()) { + if (cached_has_bits & 0x00000100u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->self_test_sample_size()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_self_test_sample_size()); } // optional int32 input_sentence_size = 11 [default = 0]; - if (has_input_sentence_size()) { + if (cached_has_bits & 0x00000200u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->input_sentence_size()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_input_sentence_size()); } // optional int32 mining_sentence_size = 12 [deprecated = true]; - if (has_mining_sentence_size()) { + if (cached_has_bits & 0x00000400u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->mining_sentence_size()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_mining_sentence_size()); } // optional int32 training_sentence_size = 13 [deprecated = true]; - if (has_training_sentence_size()) { + if (cached_has_bits & 0x00000800u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->training_sentence_size()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_training_sentence_size()); } // optional bool treat_whitespace_as_suffix = 24 [default = false]; - if (has_treat_whitespace_as_suffix()) { + if (cached_has_bits & 0x00001000u) { total_size += 2 + 1; } // optional bool split_digits = 25 [default = false]; - if (has_split_digits()) { + if (cached_has_bits & 0x00002000u) { total_size += 2 + 1; } // optional bool byte_fallback = 35 [default = false]; - if (has_byte_fallback()) { + if (cached_has_bits & 0x00004000u) { total_size += 2 + 1; } // optional bool use_all_vocab = 34 [default = false]; - if (has_use_all_vocab()) { + if (cached_has_bits & 0x00008000u) { total_size += 2 + 1; } } - if (_has_bits_[16 / 32] & 16711680u) { + if (cached_has_bits & 0x00ff0000u) { // optional int32 unk_id = 40 [default = 0]; - if (has_unk_id()) { + if (cached_has_bits & 0x00010000u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->unk_id()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_unk_id()); } // optional bool train_extremely_large_corpus = 49 [default = false]; - if (has_train_extremely_large_corpus()) { + if (cached_has_bits & 0x00020000u) { total_size += 2 + 1; } // optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default = UNIGRAM]; - if (has_model_type()) { + if (cached_has_bits & 0x00040000u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::EnumSize(this->model_type()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->_internal_model_type()); } // optional int32 vocab_size = 4 [default = 8000]; - if (has_vocab_size()) { + if (cached_has_bits & 0x00080000u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->vocab_size()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_vocab_size()); } // optional float character_coverage = 10 [default = 0.9995]; - if (has_character_coverage()) { + if (cached_has_bits & 0x00100000u) { total_size += 1 + 4; } // optional int32 seed_sentencepiece_size = 14 [default = 1000000]; - if (has_seed_sentencepiece_size()) { + if (cached_has_bits & 0x00200000u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->seed_sentencepiece_size()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_seed_sentencepiece_size()); } // optional float shrinking_factor = 15 [default = 0.75]; - if (has_shrinking_factor()) { + if (cached_has_bits & 0x00400000u) { total_size += 1 + 4; } // optional int32 num_threads = 16 [default = 16]; - if (has_num_threads()) { + if (cached_has_bits & 0x00800000u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->num_threads()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_num_threads()); } } - if (_has_bits_[24 / 32] & 4278190080u) { + if (cached_has_bits & 0xff000000u) { // optional int32 num_sub_iterations = 17 [default = 2]; - if (has_num_sub_iterations()) { + if (cached_has_bits & 0x01000000u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->num_sub_iterations()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_num_sub_iterations()); } // optional int32 max_sentence_length = 18 [default = 4192]; - if (has_max_sentence_length()) { + if (cached_has_bits & 0x02000000u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->max_sentence_length()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_max_sentence_length()); } // optional int32 max_sentencepiece_length = 20 [default = 16]; - if (has_max_sentencepiece_length()) { + if (cached_has_bits & 0x04000000u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->max_sentencepiece_length()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_max_sentencepiece_length()); } // optional bool shuffle_input_sentence = 19 [default = true]; - if (has_shuffle_input_sentence()) { + if (cached_has_bits & 0x08000000u) { total_size += 2 + 1; } // optional bool split_by_unicode_script = 21 [default = true]; - if (has_split_by_unicode_script()) { + if (cached_has_bits & 0x10000000u) { total_size += 2 + 1; } // optional bool split_by_number = 23 [default = true]; - if (has_split_by_number()) { + if (cached_has_bits & 0x20000000u) { total_size += 2 + 1; } // optional bool split_by_whitespace = 22 [default = true]; - if (has_split_by_whitespace()) { + if (cached_has_bits & 0x40000000u) { total_size += 2 + 1; } // optional bool vocabulary_output_piece_score = 32 [default = true]; - if (has_vocabulary_output_piece_score()) { + if (cached_has_bits & 0x80000000u) { total_size += 2 + 1; } } - if (_has_bits_[32 / 32] & 15u) { + cached_has_bits = _has_bits_[1]; + if (cached_has_bits & 0x0000000fu) { // optional bool hard_vocab_limit = 33 [default = true]; - if (has_hard_vocab_limit()) { + if (cached_has_bits & 0x00000001u) { total_size += 2 + 1; } // optional int32 bos_id = 41 [default = 1]; - if (has_bos_id()) { + if (cached_has_bits & 0x00000002u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->bos_id()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_bos_id()); } // optional int32 eos_id = 42 [default = 2]; - if (has_eos_id()) { + if (cached_has_bits & 0x00000004u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->eos_id()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_eos_id()); } // optional int32 pad_id = 43 [default = -1]; - if (has_pad_id()) { + if (cached_has_bits & 0x00000008u) { total_size += 2 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->pad_id()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_pad_id()); } } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void TrainerSpec::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void TrainerSpec::MergeFrom(const TrainerSpec& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.TrainerSpec) GOOGLE_DCHECK_NE(&from, this); _extensions_.MergeFrom(from._extensions_); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; input_.MergeFrom(from.input_); @@ -1578,41 +1578,33 @@ void TrainerSpec::MergeFrom(const TrainerSpec& from) { control_symbols_.MergeFrom(from.control_symbols_); user_defined_symbols_.MergeFrom(from.user_defined_symbols_); cached_has_bits = from._has_bits_[0]; - if (cached_has_bits & 255u) { + if (cached_has_bits & 0x000000ffu) { if (cached_has_bits & 0x00000001u) { - set_has_model_prefix(); - model_prefix_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.model_prefix_); + _internal_set_model_prefix(from._internal_model_prefix()); } if (cached_has_bits & 0x00000002u) { - set_has_input_format(); - input_format_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.input_format_); + _internal_set_input_format(from._internal_input_format()); } if (cached_has_bits & 0x00000004u) { - set_has_required_chars(); - required_chars_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.required_chars_); + _internal_set_required_chars(from._internal_required_chars()); } if (cached_has_bits & 0x00000008u) { - set_has_unk_surface(); - unk_surface_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), from.unk_surface_); + _internal_set_unk_surface(from._internal_unk_surface()); } if (cached_has_bits & 0x00000010u) { - set_has_unk_piece(); - unk_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), from.unk_piece_); + _internal_set_unk_piece(from._internal_unk_piece()); } if (cached_has_bits & 0x00000020u) { - set_has_bos_piece(); - bos_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), from.bos_piece_); + _internal_set_bos_piece(from._internal_bos_piece()); } if (cached_has_bits & 0x00000040u) { - set_has_eos_piece(); - eos_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), from.eos_piece_); + _internal_set_eos_piece(from._internal_eos_piece()); } if (cached_has_bits & 0x00000080u) { - set_has_pad_piece(); - pad_piece_.AssignWithDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), from.pad_piece_); + _internal_set_pad_piece(from._internal_pad_piece()); } } - if (cached_has_bits & 65280u) { + if (cached_has_bits & 0x0000ff00u) { if (cached_has_bits & 0x00000100u) { self_test_sample_size_ = from.self_test_sample_size_; } @@ -1639,7 +1631,7 @@ void TrainerSpec::MergeFrom(const TrainerSpec& from) { } _has_bits_[0] |= cached_has_bits; } - if (cached_has_bits & 16711680u) { + if (cached_has_bits & 0x00ff0000u) { if (cached_has_bits & 0x00010000u) { unk_id_ = from.unk_id_; } @@ -1666,7 +1658,7 @@ void TrainerSpec::MergeFrom(const TrainerSpec& from) { } _has_bits_[0] |= cached_has_bits; } - if (cached_has_bits & 4278190080u) { + if (cached_has_bits & 0xff000000u) { if (cached_has_bits & 0x01000000u) { num_sub_iterations_ = from.num_sub_iterations_; } @@ -1694,7 +1686,7 @@ void TrainerSpec::MergeFrom(const TrainerSpec& from) { _has_bits_[0] |= cached_has_bits; } cached_has_bits = from._has_bits_[1]; - if (cached_has_bits & 15u) { + if (cached_has_bits & 0x0000000fu) { if (cached_has_bits & 0x00000001u) { hard_vocab_limit_ = from.hard_vocab_limit_; } @@ -1726,42 +1718,30 @@ bool TrainerSpec::IsInitialized() const { return true; } -void TrainerSpec::Swap(TrainerSpec* other) { - if (other == this) return; - InternalSwap(other); -} void TrainerSpec::InternalSwap(TrainerSpec* other) { using std::swap; - input_.InternalSwap(CastToBase(&other->input_)); - accept_language_.InternalSwap(CastToBase(&other->accept_language_)); - control_symbols_.InternalSwap(CastToBase(&other->control_symbols_)); - user_defined_symbols_.InternalSwap(CastToBase(&other->user_defined_symbols_)); - model_prefix_.Swap(&other->model_prefix_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - input_format_.Swap(&other->input_format_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - required_chars_.Swap(&other->required_chars_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - unk_surface_.Swap(&other->unk_surface_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), - GetArenaNoVirtual()); - unk_piece_.Swap(&other->unk_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), - GetArenaNoVirtual()); - bos_piece_.Swap(&other->bos_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), - GetArenaNoVirtual()); - eos_piece_.Swap(&other->eos_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), - GetArenaNoVirtual()); - pad_piece_.Swap(&other->pad_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), - GetArenaNoVirtual()); - swap(self_test_sample_size_, other->self_test_sample_size_); - swap(input_sentence_size_, other->input_sentence_size_); - swap(mining_sentence_size_, other->mining_sentence_size_); - swap(training_sentence_size_, other->training_sentence_size_); - swap(treat_whitespace_as_suffix_, other->treat_whitespace_as_suffix_); - swap(split_digits_, other->split_digits_); - swap(byte_fallback_, other->byte_fallback_); - swap(use_all_vocab_, other->use_all_vocab_); - swap(unk_id_, other->unk_id_); - swap(train_extremely_large_corpus_, other->train_extremely_large_corpus_); + _extensions_.Swap(&other->_extensions_); + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(_has_bits_[0], other->_has_bits_[0]); + swap(_has_bits_[1], other->_has_bits_[1]); + input_.InternalSwap(&other->input_); + accept_language_.InternalSwap(&other->accept_language_); + control_symbols_.InternalSwap(&other->control_symbols_); + user_defined_symbols_.InternalSwap(&other->user_defined_symbols_); + model_prefix_.Swap(&other->model_prefix_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + input_format_.Swap(&other->input_format_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + required_chars_.Swap(&other->required_chars_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + unk_surface_.Swap(&other->unk_surface_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), GetArena()); + unk_piece_.Swap(&other->unk_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), GetArena()); + bos_piece_.Swap(&other->bos_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), GetArena()); + eos_piece_.Swap(&other->eos_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), GetArena()); + pad_piece_.Swap(&other->pad_piece_, &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), GetArena()); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(TrainerSpec, train_extremely_large_corpus_) + + sizeof(TrainerSpec::train_extremely_large_corpus_) + - PROTOBUF_FIELD_OFFSET(TrainerSpec, self_test_sample_size_)>( + reinterpret_cast(&self_test_sample_size_), + reinterpret_cast(&other->self_test_sample_size_)); swap(model_type_, other->model_type_); swap(vocab_size_, other->vocab_size_); swap(character_coverage_, other->character_coverage_); @@ -1780,13 +1760,9 @@ void TrainerSpec::InternalSwap(TrainerSpec* other) { swap(bos_id_, other->bos_id_); swap(eos_id_, other->eos_id_); swap(pad_id_, other->pad_id_); - swap(_has_bits_[0], other->_has_bits_[0]); - swap(_has_bits_[1], other->_has_bits_[1]); - _internal_metadata_.Swap(&other->_internal_metadata_); - _extensions_.Swap(&other->_extensions_); } -::std::string TrainerSpec::GetTypeName() const { +std::string TrainerSpec::GetTypeName() const { return "sentencepiece.TrainerSpec"; } @@ -1795,39 +1771,55 @@ ::std::string TrainerSpec::GetTypeName() const { void NormalizerSpec::InitAsDefaultInstance() { } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int NormalizerSpec::kNameFieldNumber; -const int NormalizerSpec::kPrecompiledCharsmapFieldNumber; -const int NormalizerSpec::kAddDummyPrefixFieldNumber; -const int NormalizerSpec::kRemoveExtraWhitespacesFieldNumber; -const int NormalizerSpec::kEscapeWhitespacesFieldNumber; -const int NormalizerSpec::kNormalizationRuleTsvFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -NormalizerSpec::NormalizerSpec() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_NormalizerSpec.base); +class NormalizerSpec::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static void set_has_name(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static void set_has_precompiled_charsmap(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } + static void set_has_add_dummy_prefix(HasBits* has_bits) { + (*has_bits)[0] |= 8u; + } + static void set_has_remove_extra_whitespaces(HasBits* has_bits) { + (*has_bits)[0] |= 16u; + } + static void set_has_escape_whitespaces(HasBits* has_bits) { + (*has_bits)[0] |= 32u; + } + static void set_has_normalization_rule_tsv(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } +}; + +NormalizerSpec::NormalizerSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + _extensions_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.NormalizerSpec) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.NormalizerSpec) } NormalizerSpec::NormalizerSpec(const NormalizerSpec& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); _extensions_.MergeFrom(from._extensions_); - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_name()) { - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); + name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_name()) { + name_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_name(), + GetArena()); } - precompiled_charsmap_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_precompiled_charsmap()) { - precompiled_charsmap_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.precompiled_charsmap_); + precompiled_charsmap_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_precompiled_charsmap()) { + precompiled_charsmap_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_precompiled_charsmap(), + GetArena()); } - normalization_rule_tsv_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_normalization_rule_tsv()) { - normalization_rule_tsv_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.normalization_rule_tsv_); + normalization_rule_tsv_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_normalization_rule_tsv()) { + normalization_rule_tsv_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_normalization_rule_tsv(), + GetArena()); } ::memcpy(&add_dummy_prefix_, &from.add_dummy_prefix_, static_cast(reinterpret_cast(&escape_whitespaces_) - @@ -1836,9 +1828,10 @@ NormalizerSpec::NormalizerSpec(const NormalizerSpec& from) } void NormalizerSpec::SharedCtor() { - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - precompiled_charsmap_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - normalization_rule_tsv_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto.base); + name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + precompiled_charsmap_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + normalization_rule_tsv_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); add_dummy_prefix_ = true; remove_extra_whitespaces_ = true; escape_whitespaces_ = true; @@ -1847,216 +1840,196 @@ void NormalizerSpec::SharedCtor() { NormalizerSpec::~NormalizerSpec() { // @@protoc_insertion_point(destructor:sentencepiece.NormalizerSpec) SharedDtor(); + _internal_metadata_.Delete(); } void NormalizerSpec::SharedDtor() { - name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - precompiled_charsmap_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - normalization_rule_tsv_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + GOOGLE_DCHECK(GetArena() == nullptr); + name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + precompiled_charsmap_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + normalization_rule_tsv_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } +void NormalizerSpec::ArenaDtor(void* object) { + NormalizerSpec* _this = reinterpret_cast< NormalizerSpec* >(object); + (void)_this; +} +void NormalizerSpec::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void NormalizerSpec::SetCachedSize(int size) const { _cached_size_.Set(size); } const NormalizerSpec& NormalizerSpec::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_5fmodel_2eproto::scc_info_NormalizerSpec.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto.base); return *internal_default_instance(); } void NormalizerSpec::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.NormalizerSpec) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _extensions_.Clear(); cached_has_bits = _has_bits_[0]; - if (cached_has_bits & 63u) { + if (cached_has_bits & 0x0000003fu) { if (cached_has_bits & 0x00000001u) { - name_.ClearNonDefaultToEmptyNoArena(); + name_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000002u) { - precompiled_charsmap_.ClearNonDefaultToEmptyNoArena(); + precompiled_charsmap_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000004u) { - normalization_rule_tsv_.ClearNonDefaultToEmptyNoArena(); + normalization_rule_tsv_.ClearNonDefaultToEmpty(); } add_dummy_prefix_ = true; remove_extra_whitespaces_ = true; escape_whitespaces_ = true; } _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool NormalizerSpec::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.NormalizerSpec) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + _internal_metadata_.Clear(); +} + +const char* NormalizerSpec::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { // optional string name = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - } else { - goto handle_unusual; - } - break; - } - + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + auto str = _internal_mutable_name(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bytes precompiled_charsmap = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( - input, this->mutable_precompiled_charsmap())); - } else { - goto handle_unusual; - } - break; - } - + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + auto str = _internal_mutable_precompiled_charsmap(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool add_dummy_prefix = 3 [default = true]; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(24u /* 24 & 0xFF */)) { - set_has_add_dummy_prefix(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &add_dummy_prefix_))); - } else { - goto handle_unusual; - } - break; - } - + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + _Internal::set_has_add_dummy_prefix(&has_bits); + add_dummy_prefix_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool remove_extra_whitespaces = 4 [default = true]; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(32u /* 32 & 0xFF */)) { - set_has_remove_extra_whitespaces(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &remove_extra_whitespaces_))); - } else { - goto handle_unusual; - } - break; - } - + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { + _Internal::set_has_remove_extra_whitespaces(&has_bits); + remove_extra_whitespaces_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional bool escape_whitespaces = 5 [default = true]; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(40u /* 40 & 0xFF */)) { - set_has_escape_whitespaces(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &escape_whitespaces_))); - } else { - goto handle_unusual; - } - break; - } - + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) { + _Internal::set_has_escape_whitespaces(&has_bits); + escape_whitespaces_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string normalization_rule_tsv = 6; - case 6: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_normalization_rule_tsv())); - } else { - goto handle_unusual; - } - break; - } - + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) { + auto str = _internal_mutable_normalization_rule_tsv(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - if ((1600u <= tag)) { - DO_(_extensions_.ParseField(tag, input, - internal_default_instance(), - &unknown_fields_stream)); - continue; - } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + if ((1600u <= tag)) { + ptr = _extensions_.ParseField(tag, ptr, + internal_default_instance(), &_internal_metadata_, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.NormalizerSpec) - return true; + _has_bits_.Or(has_bits); + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.NormalizerSpec) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void NormalizerSpec::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.NormalizerSpec) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* NormalizerSpec::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.NormalizerSpec) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = _has_bits_[0]; // optional string name = 1; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); + target = stream->WriteStringMaybeAliased( + 1, this->_internal_name(), target); } // optional bytes precompiled_charsmap = 2; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( - 2, this->precompiled_charsmap(), output); + target = stream->WriteBytesMaybeAliased( + 2, this->_internal_precompiled_charsmap(), target); } // optional bool add_dummy_prefix = 3 [default = true]; if (cached_has_bits & 0x00000008u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(3, this->add_dummy_prefix(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(3, this->_internal_add_dummy_prefix(), target); } // optional bool remove_extra_whitespaces = 4 [default = true]; if (cached_has_bits & 0x00000010u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(4, this->remove_extra_whitespaces(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(4, this->_internal_remove_extra_whitespaces(), target); } // optional bool escape_whitespaces = 5 [default = true]; if (cached_has_bits & 0x00000020u) { - ::google::protobuf::internal::WireFormatLite::WriteBool(5, this->escape_whitespaces(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(5, this->_internal_escape_whitespaces(), target); } // optional string normalization_rule_tsv = 6; if (cached_has_bits & 0x00000004u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 6, this->normalization_rule_tsv(), output); + target = stream->WriteStringMaybeAliased( + 6, this->_internal_normalization_rule_tsv(), target); } // Extension range [200, 536870912) - _extensions_.SerializeWithCachedSizes( - 200, 536870912, output); + target = _extensions_._InternalSerialize( + 200, 536870912, target, stream); - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.NormalizerSpec) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.NormalizerSpec) + return target; } size_t NormalizerSpec::ByteSizeLong() const { @@ -2065,77 +2038,81 @@ size_t NormalizerSpec::ByteSizeLong() const { total_size += _extensions_.ByteSize(); - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; - if (_has_bits_[0 / 32] & 63u) { + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x0000003fu) { // optional string name = 1; - if (has_name()) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_name()); } // optional bytes precompiled_charsmap = 2; - if (has_precompiled_charsmap()) { + if (cached_has_bits & 0x00000002u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::BytesSize( - this->precompiled_charsmap()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize( + this->_internal_precompiled_charsmap()); } // optional string normalization_rule_tsv = 6; - if (has_normalization_rule_tsv()) { + if (cached_has_bits & 0x00000004u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->normalization_rule_tsv()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_normalization_rule_tsv()); } // optional bool add_dummy_prefix = 3 [default = true]; - if (has_add_dummy_prefix()) { + if (cached_has_bits & 0x00000008u) { total_size += 1 + 1; } // optional bool remove_extra_whitespaces = 4 [default = true]; - if (has_remove_extra_whitespaces()) { + if (cached_has_bits & 0x00000010u) { total_size += 1 + 1; } // optional bool escape_whitespaces = 5 [default = true]; - if (has_escape_whitespaces()) { + if (cached_has_bits & 0x00000020u) { total_size += 1 + 1; } } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void NormalizerSpec::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void NormalizerSpec::MergeFrom(const NormalizerSpec& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.NormalizerSpec) GOOGLE_DCHECK_NE(&from, this); _extensions_.MergeFrom(from._extensions_); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = from._has_bits_[0]; - if (cached_has_bits & 63u) { + if (cached_has_bits & 0x0000003fu) { if (cached_has_bits & 0x00000001u) { - set_has_name(); - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); + _internal_set_name(from._internal_name()); } if (cached_has_bits & 0x00000002u) { - set_has_precompiled_charsmap(); - precompiled_charsmap_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.precompiled_charsmap_); + _internal_set_precompiled_charsmap(from._internal_precompiled_charsmap()); } if (cached_has_bits & 0x00000004u) { - set_has_normalization_rule_tsv(); - normalization_rule_tsv_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.normalization_rule_tsv_); + _internal_set_normalization_rule_tsv(from._internal_normalization_rule_tsv()); } if (cached_has_bits & 0x00000008u) { add_dummy_prefix_ = from.add_dummy_prefix_; @@ -2165,27 +2142,20 @@ bool NormalizerSpec::IsInitialized() const { return true; } -void NormalizerSpec::Swap(NormalizerSpec* other) { - if (other == this) return; - InternalSwap(other); -} void NormalizerSpec::InternalSwap(NormalizerSpec* other) { using std::swap; - name_.Swap(&other->name_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - precompiled_charsmap_.Swap(&other->precompiled_charsmap_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - normalization_rule_tsv_.Swap(&other->normalization_rule_tsv_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); + _extensions_.Swap(&other->_extensions_); + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(_has_bits_[0], other->_has_bits_[0]); + name_.Swap(&other->name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + precompiled_charsmap_.Swap(&other->precompiled_charsmap_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + normalization_rule_tsv_.Swap(&other->normalization_rule_tsv_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); swap(add_dummy_prefix_, other->add_dummy_prefix_); swap(remove_extra_whitespaces_, other->remove_extra_whitespaces_); swap(escape_whitespaces_, other->escape_whitespaces_); - swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); - _extensions_.Swap(&other->_extensions_); } -::std::string NormalizerSpec::GetTypeName() const { +std::string NormalizerSpec::GetTypeName() const { return "sentencepiece.NormalizerSpec"; } @@ -2194,209 +2164,220 @@ ::std::string NormalizerSpec::GetTypeName() const { void SelfTestData_Sample::InitAsDefaultInstance() { } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int SelfTestData_Sample::kInputFieldNumber; -const int SelfTestData_Sample::kExpectedFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 +class SelfTestData_Sample::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static void set_has_input(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static void set_has_expected(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } +}; -SelfTestData_Sample::SelfTestData_Sample() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_SelfTestData_Sample.base); +SelfTestData_Sample::SelfTestData_Sample(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.SelfTestData.Sample) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.SelfTestData.Sample) } SelfTestData_Sample::SelfTestData_Sample(const SelfTestData_Sample& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - input_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_input()) { - input_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.input_); + _internal_metadata_.MergeFrom(from._internal_metadata_); + input_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_input()) { + input_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_input(), + GetArena()); } - expected_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_expected()) { - expected_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.expected_); + expected_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_expected()) { + expected_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_expected(), + GetArena()); } // @@protoc_insertion_point(copy_constructor:sentencepiece.SelfTestData.Sample) } void SelfTestData_Sample::SharedCtor() { - input_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - expected_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto.base); + input_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + expected_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } SelfTestData_Sample::~SelfTestData_Sample() { // @@protoc_insertion_point(destructor:sentencepiece.SelfTestData.Sample) SharedDtor(); + _internal_metadata_.Delete(); } void SelfTestData_Sample::SharedDtor() { - input_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - expected_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + GOOGLE_DCHECK(GetArena() == nullptr); + input_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + expected_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } +void SelfTestData_Sample::ArenaDtor(void* object) { + SelfTestData_Sample* _this = reinterpret_cast< SelfTestData_Sample* >(object); + (void)_this; +} +void SelfTestData_Sample::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void SelfTestData_Sample::SetCachedSize(int size) const { _cached_size_.Set(size); } const SelfTestData_Sample& SelfTestData_Sample::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_5fmodel_2eproto::scc_info_SelfTestData_Sample.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto.base); return *internal_default_instance(); } void SelfTestData_Sample::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.SelfTestData.Sample) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; cached_has_bits = _has_bits_[0]; - if (cached_has_bits & 3u) { + if (cached_has_bits & 0x00000003u) { if (cached_has_bits & 0x00000001u) { - input_.ClearNonDefaultToEmptyNoArena(); + input_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000002u) { - expected_.ClearNonDefaultToEmptyNoArena(); + expected_.ClearNonDefaultToEmpty(); } } _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool SelfTestData_Sample::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.SelfTestData.Sample) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + _internal_metadata_.Clear(); +} + +const char* SelfTestData_Sample::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { // optional string input = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_input())); - } else { - goto handle_unusual; - } - break; - } - + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + auto str = _internal_mutable_input(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional string expected = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_expected())); - } else { - goto handle_unusual; - } - break; - } - + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + auto str = _internal_mutable_expected(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.SelfTestData.Sample) - return true; + _has_bits_.Or(has_bits); + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.SelfTestData.Sample) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void SelfTestData_Sample::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.SelfTestData.Sample) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* SelfTestData_Sample::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.SelfTestData.Sample) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = _has_bits_[0]; // optional string input = 1; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->input(), output); + target = stream->WriteStringMaybeAliased( + 1, this->_internal_input(), target); } // optional string expected = 2; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->expected(), output); + target = stream->WriteStringMaybeAliased( + 2, this->_internal_expected(), target); } - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.SelfTestData.Sample) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.SelfTestData.Sample) + return target; } size_t SelfTestData_Sample::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:sentencepiece.SelfTestData.Sample) size_t total_size = 0; - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; - if (_has_bits_[0 / 32] & 3u) { + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x00000003u) { // optional string input = 1; - if (has_input()) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->input()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_input()); } // optional string expected = 2; - if (has_expected()) { + if (cached_has_bits & 0x00000002u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->expected()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_expected()); } } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void SelfTestData_Sample::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void SelfTestData_Sample::MergeFrom(const SelfTestData_Sample& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.SelfTestData.Sample) GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = from._has_bits_[0]; - if (cached_has_bits & 3u) { + if (cached_has_bits & 0x00000003u) { if (cached_has_bits & 0x00000001u) { - set_has_input(); - input_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.input_); + _internal_set_input(from._internal_input()); } if (cached_has_bits & 0x00000002u) { - set_has_expected(); - expected_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.expected_); + _internal_set_expected(from._internal_expected()); } } } @@ -2412,21 +2393,15 @@ bool SelfTestData_Sample::IsInitialized() const { return true; } -void SelfTestData_Sample::Swap(SelfTestData_Sample* other) { - if (other == this) return; - InternalSwap(other); -} void SelfTestData_Sample::InternalSwap(SelfTestData_Sample* other) { using std::swap; - input_.Swap(&other->input_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); - expected_.Swap(&other->expected_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); + _internal_metadata_.Swap(&other->_internal_metadata_); swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); + input_.Swap(&other->input_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + expected_.Swap(&other->expected_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -::std::string SelfTestData_Sample::GetTypeName() const { +std::string SelfTestData_Sample::GetTypeName() const { return "sentencepiece.SelfTestData.Sample"; } @@ -2435,135 +2410,138 @@ ::std::string SelfTestData_Sample::GetTypeName() const { void SelfTestData::InitAsDefaultInstance() { } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int SelfTestData::kSamplesFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 +class SelfTestData::_Internal { + public: +}; -SelfTestData::SelfTestData() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_SelfTestData.base); +SelfTestData::SelfTestData(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + _extensions_(arena), + samples_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.SelfTestData) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.SelfTestData) } SelfTestData::SelfTestData(const SelfTestData& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), - _has_bits_(from._has_bits_), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), samples_(from.samples_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); _extensions_.MergeFrom(from._extensions_); // @@protoc_insertion_point(copy_constructor:sentencepiece.SelfTestData) } void SelfTestData::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SelfTestData_sentencepiece_5fmodel_2eproto.base); } SelfTestData::~SelfTestData() { // @@protoc_insertion_point(destructor:sentencepiece.SelfTestData) SharedDtor(); + _internal_metadata_.Delete(); } void SelfTestData::SharedDtor() { + GOOGLE_DCHECK(GetArena() == nullptr); } +void SelfTestData::ArenaDtor(void* object) { + SelfTestData* _this = reinterpret_cast< SelfTestData* >(object); + (void)_this; +} +void SelfTestData::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void SelfTestData::SetCachedSize(int size) const { _cached_size_.Set(size); } const SelfTestData& SelfTestData::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_5fmodel_2eproto::scc_info_SelfTestData.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_SelfTestData_sentencepiece_5fmodel_2eproto.base); return *internal_default_instance(); } void SelfTestData::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.SelfTestData) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _extensions_.Clear(); samples_.Clear(); - _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool SelfTestData::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.SelfTestData) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // repeated .sentencepiece.SelfTestData.Sample samples = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, add_samples())); - } else { - goto handle_unusual; - } - break; - } + _internal_metadata_.Clear(); +} +const char* SelfTestData::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // repeated .sentencepiece.SelfTestData.Sample samples = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(_internal_add_samples(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - if ((1600u <= tag)) { - DO_(_extensions_.ParseField(tag, input, - internal_default_instance(), - &unknown_fields_stream)); - continue; - } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + if ((1600u <= tag)) { + ptr = _extensions_.ParseField(tag, ptr, + internal_default_instance(), &_internal_metadata_, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.SelfTestData) - return true; + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.SelfTestData) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void SelfTestData::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.SelfTestData) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* SelfTestData::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.SelfTestData) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; // repeated .sentencepiece.SelfTestData.Sample samples = 1; for (unsigned int i = 0, - n = static_cast(this->samples_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 1, - this->samples(static_cast(i)), - output); + n = static_cast(this->_internal_samples_size()); i < n; i++) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(1, this->_internal_samples(i), target, stream); } // Extension range [200, 536870912) - _extensions_.SerializeWithCachedSizes( - 200, 536870912, output); + target = _extensions_._InternalSerialize( + 200, 536870912, target, stream); - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.SelfTestData) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.SelfTestData) + return target; } size_t SelfTestData::ByteSizeLong() const { @@ -2572,35 +2550,37 @@ size_t SelfTestData::ByteSizeLong() const { total_size += _extensions_.ByteSize(); - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; // repeated .sentencepiece.SelfTestData.Sample samples = 1; - { - unsigned int count = static_cast(this->samples_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->samples(static_cast(i))); - } + total_size += 1UL * this->_internal_samples_size(); + for (const auto& msg : this->samples_) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void SelfTestData::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void SelfTestData::MergeFrom(const SelfTestData& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.SelfTestData) GOOGLE_DCHECK_NE(&from, this); _extensions_.MergeFrom(from._extensions_); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; samples_.MergeFrom(from.samples_); @@ -2621,19 +2601,14 @@ bool SelfTestData::IsInitialized() const { return true; } -void SelfTestData::Swap(SelfTestData* other) { - if (other == this) return; - InternalSwap(other); -} void SelfTestData::InternalSwap(SelfTestData* other) { using std::swap; - CastToBase(&samples_)->InternalSwap(CastToBase(&other->samples_)); - swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); _extensions_.Swap(&other->_extensions_); + _internal_metadata_.Swap(&other->_internal_metadata_); + samples_.InternalSwap(&other->samples_); } -::std::string SelfTestData::GetTypeName() const { +std::string SelfTestData::GetTypeName() const { return "sentencepiece.SelfTestData"; } @@ -2642,28 +2617,36 @@ ::std::string SelfTestData::GetTypeName() const { void ModelProto_SentencePiece::InitAsDefaultInstance() { } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int ModelProto_SentencePiece::kPieceFieldNumber; -const int ModelProto_SentencePiece::kScoreFieldNumber; -const int ModelProto_SentencePiece::kTypeFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 +class ModelProto_SentencePiece::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static void set_has_piece(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static void set_has_score(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } + static void set_has_type(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } +}; -ModelProto_SentencePiece::ModelProto_SentencePiece() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_ModelProto_SentencePiece.base); +ModelProto_SentencePiece::ModelProto_SentencePiece(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + _extensions_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.ModelProto.SentencePiece) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.ModelProto.SentencePiece) } ModelProto_SentencePiece::ModelProto_SentencePiece(const ModelProto_SentencePiece& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); _extensions_.MergeFrom(from._extensions_); - piece_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.has_piece()) { - piece_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.piece_); + piece_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (from._internal_has_piece()) { + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_piece(), + GetArena()); } ::memcpy(&score_, &from.score_, static_cast(reinterpret_cast(&type_) - @@ -2672,7 +2655,8 @@ ModelProto_SentencePiece::ModelProto_SentencePiece(const ModelProto_SentencePiec } void ModelProto_SentencePiece::SharedCtor() { - piece_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto.base); + piece_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); score_ = 0; type_ = 1; } @@ -2680,160 +2664,150 @@ void ModelProto_SentencePiece::SharedCtor() { ModelProto_SentencePiece::~ModelProto_SentencePiece() { // @@protoc_insertion_point(destructor:sentencepiece.ModelProto.SentencePiece) SharedDtor(); + _internal_metadata_.Delete(); } void ModelProto_SentencePiece::SharedDtor() { - piece_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + GOOGLE_DCHECK(GetArena() == nullptr); + piece_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } +void ModelProto_SentencePiece::ArenaDtor(void* object) { + ModelProto_SentencePiece* _this = reinterpret_cast< ModelProto_SentencePiece* >(object); + (void)_this; +} +void ModelProto_SentencePiece::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void ModelProto_SentencePiece::SetCachedSize(int size) const { _cached_size_.Set(size); } const ModelProto_SentencePiece& ModelProto_SentencePiece::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_5fmodel_2eproto::scc_info_ModelProto_SentencePiece.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto.base); return *internal_default_instance(); } void ModelProto_SentencePiece::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.ModelProto.SentencePiece) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _extensions_.Clear(); cached_has_bits = _has_bits_[0]; if (cached_has_bits & 0x00000001u) { - piece_.ClearNonDefaultToEmptyNoArena(); + piece_.ClearNonDefaultToEmpty(); } - if (cached_has_bits & 6u) { + if (cached_has_bits & 0x00000006u) { score_ = 0; type_ = 1; } _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool ModelProto_SentencePiece::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.ModelProto.SentencePiece) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + _internal_metadata_.Clear(); +} + +const char* ModelProto_SentencePiece::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { // optional string piece = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_piece())); - } else { - goto handle_unusual; - } - break; - } - + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + auto str = _internal_mutable_piece(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional float score = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(21u /* 21 & 0xFF */)) { - set_has_score(); - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( - input, &score_))); - } else { - goto handle_unusual; - } - break; - } - + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 21)) { + _Internal::set_has_score(&has_bits); + score_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(float); + } else goto handle_unusual; + continue; // optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default = NORMAL]; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(24u /* 24 & 0xFF */)) { - int value; - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( - input, &value))); - if (::sentencepiece::ModelProto_SentencePiece_Type_IsValid(value)) { - set_type(static_cast< ::sentencepiece::ModelProto_SentencePiece_Type >(value)); + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + ::PROTOBUF_NAMESPACE_ID::uint64 val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + if (PROTOBUF_PREDICT_TRUE(::sentencepiece::ModelProto_SentencePiece_Type_IsValid(val))) { + _internal_set_type(static_cast<::sentencepiece::ModelProto_SentencePiece_Type>(val)); } else { - unknown_fields_stream.WriteVarint32(24u); - unknown_fields_stream.WriteVarint32( - static_cast< ::google::protobuf::uint32>(value)); + ::PROTOBUF_NAMESPACE_ID::internal::WriteVarint(3, val, mutable_unknown_fields()); } - } else { - goto handle_unusual; - } - break; - } - + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - if ((1600u <= tag)) { - DO_(_extensions_.ParseField(tag, input, - internal_default_instance(), - &unknown_fields_stream)); - continue; - } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + if ((1600u <= tag)) { + ptr = _extensions_.ParseField(tag, ptr, + internal_default_instance(), &_internal_metadata_, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.ModelProto.SentencePiece) - return true; + _has_bits_.Or(has_bits); + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.ModelProto.SentencePiece) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void ModelProto_SentencePiece::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.ModelProto.SentencePiece) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* ModelProto_SentencePiece::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.ModelProto.SentencePiece) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = _has_bits_[0]; // optional string piece = 1; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->piece(), output); + target = stream->WriteStringMaybeAliased( + 1, this->_internal_piece(), target); } // optional float score = 2; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteFloat(2, this->score(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(2, this->_internal_score(), target); } // optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default = NORMAL]; if (cached_has_bits & 0x00000004u) { - ::google::protobuf::internal::WireFormatLite::WriteEnum( - 3, this->type(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray( + 3, this->_internal_type(), target); } // Extension range [200, 536870912) - _extensions_.SerializeWithCachedSizes( - 200, 536870912, output); + target = _extensions_._InternalSerialize( + 200, 536870912, target, stream); - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.ModelProto.SentencePiece) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.ModelProto.SentencePiece) + return target; } size_t ModelProto_SentencePiece::ByteSizeLong() const { @@ -2842,51 +2816,57 @@ size_t ModelProto_SentencePiece::ByteSizeLong() const { total_size += _extensions_.ByteSize(); - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; - if (_has_bits_[0 / 32] & 7u) { + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x00000007u) { // optional string piece = 1; - if (has_piece()) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->piece()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_piece()); } // optional float score = 2; - if (has_score()) { + if (cached_has_bits & 0x00000002u) { total_size += 1 + 4; } // optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default = NORMAL]; - if (has_type()) { + if (cached_has_bits & 0x00000004u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::EnumSize(this->type()); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->_internal_type()); } } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void ModelProto_SentencePiece::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void ModelProto_SentencePiece::MergeFrom(const ModelProto_SentencePiece& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.ModelProto.SentencePiece) GOOGLE_DCHECK_NE(&from, this); _extensions_.MergeFrom(from._extensions_); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; cached_has_bits = from._has_bits_[0]; - if (cached_has_bits & 7u) { + if (cached_has_bits & 0x00000007u) { if (cached_has_bits & 0x00000001u) { - set_has_piece(); - piece_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.piece_); + _internal_set_piece(from._internal_piece()); } if (cached_has_bits & 0x00000002u) { score_ = from.score_; @@ -2913,22 +2893,17 @@ bool ModelProto_SentencePiece::IsInitialized() const { return true; } -void ModelProto_SentencePiece::Swap(ModelProto_SentencePiece* other) { - if (other == this) return; - InternalSwap(other); -} void ModelProto_SentencePiece::InternalSwap(ModelProto_SentencePiece* other) { using std::swap; - piece_.Swap(&other->piece_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), - GetArenaNoVirtual()); + _extensions_.Swap(&other->_extensions_); + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(_has_bits_[0], other->_has_bits_[0]); + piece_.Swap(&other->piece_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); swap(score_, other->score_); swap(type_, other->type_); - swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); - _extensions_.Swap(&other->_extensions_); } -::std::string ModelProto_SentencePiece::GetTypeName() const { +std::string ModelProto_SentencePiece::GetTypeName() const { return "sentencepiece.ModelProto.SentencePiece"; } @@ -2945,52 +2920,82 @@ void ModelProto::InitAsDefaultInstance() { ::sentencepiece::_ModelProto_default_instance_._instance.get_mutable()->denormalizer_spec_ = const_cast< ::sentencepiece::NormalizerSpec*>( ::sentencepiece::NormalizerSpec::internal_default_instance()); } -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int ModelProto::kPiecesFieldNumber; -const int ModelProto::kTrainerSpecFieldNumber; -const int ModelProto::kNormalizerSpecFieldNumber; -const int ModelProto::kSelfTestDataFieldNumber; -const int ModelProto::kDenormalizerSpecFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -ModelProto::ModelProto() - : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { - ::google::protobuf::internal::InitSCC( - &protobuf_sentencepiece_5fmodel_2eproto::scc_info_ModelProto.base); +class ModelProto::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static const ::sentencepiece::TrainerSpec& trainer_spec(const ModelProto* msg); + static void set_has_trainer_spec(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static const ::sentencepiece::NormalizerSpec& normalizer_spec(const ModelProto* msg); + static void set_has_normalizer_spec(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } + static const ::sentencepiece::SelfTestData& self_test_data(const ModelProto* msg); + static void set_has_self_test_data(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } + static const ::sentencepiece::NormalizerSpec& denormalizer_spec(const ModelProto* msg); + static void set_has_denormalizer_spec(HasBits* has_bits) { + (*has_bits)[0] |= 8u; + } +}; + +const ::sentencepiece::TrainerSpec& +ModelProto::_Internal::trainer_spec(const ModelProto* msg) { + return *msg->trainer_spec_; +} +const ::sentencepiece::NormalizerSpec& +ModelProto::_Internal::normalizer_spec(const ModelProto* msg) { + return *msg->normalizer_spec_; +} +const ::sentencepiece::SelfTestData& +ModelProto::_Internal::self_test_data(const ModelProto* msg) { + return *msg->self_test_data_; +} +const ::sentencepiece::NormalizerSpec& +ModelProto::_Internal::denormalizer_spec(const ModelProto* msg) { + return *msg->denormalizer_spec_; +} +ModelProto::ModelProto(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), + _extensions_(arena), + pieces_(arena) { SharedCtor(); - // @@protoc_insertion_point(constructor:sentencepiece.ModelProto) + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:sentencepiece.ModelProto) } ModelProto::ModelProto(const ModelProto& from) - : ::google::protobuf::MessageLite(), - _internal_metadata_(NULL), + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_), pieces_(from.pieces_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); + _internal_metadata_.MergeFrom(from._internal_metadata_); _extensions_.MergeFrom(from._extensions_); - if (from.has_trainer_spec()) { + if (from._internal_has_trainer_spec()) { trainer_spec_ = new ::sentencepiece::TrainerSpec(*from.trainer_spec_); } else { - trainer_spec_ = NULL; + trainer_spec_ = nullptr; } - if (from.has_normalizer_spec()) { + if (from._internal_has_normalizer_spec()) { normalizer_spec_ = new ::sentencepiece::NormalizerSpec(*from.normalizer_spec_); } else { - normalizer_spec_ = NULL; + normalizer_spec_ = nullptr; } - if (from.has_self_test_data()) { + if (from._internal_has_self_test_data()) { self_test_data_ = new ::sentencepiece::SelfTestData(*from.self_test_data_); } else { - self_test_data_ = NULL; + self_test_data_ = nullptr; } - if (from.has_denormalizer_spec()) { + if (from._internal_has_denormalizer_spec()) { denormalizer_spec_ = new ::sentencepiece::NormalizerSpec(*from.denormalizer_spec_); } else { - denormalizer_spec_ = NULL; + denormalizer_spec_ = nullptr; } // @@protoc_insertion_point(copy_constructor:sentencepiece.ModelProto) } void ModelProto::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_ModelProto_sentencepiece_5fmodel_2eproto.base); ::memset(&trainer_spec_, 0, static_cast( reinterpret_cast(&denormalizer_spec_) - reinterpret_cast(&trainer_spec_)) + sizeof(denormalizer_spec_)); @@ -2999,204 +3004,198 @@ void ModelProto::SharedCtor() { ModelProto::~ModelProto() { // @@protoc_insertion_point(destructor:sentencepiece.ModelProto) SharedDtor(); + _internal_metadata_.Delete(); } void ModelProto::SharedDtor() { + GOOGLE_DCHECK(GetArena() == nullptr); if (this != internal_default_instance()) delete trainer_spec_; if (this != internal_default_instance()) delete normalizer_spec_; if (this != internal_default_instance()) delete self_test_data_; if (this != internal_default_instance()) delete denormalizer_spec_; } +void ModelProto::ArenaDtor(void* object) { + ModelProto* _this = reinterpret_cast< ModelProto* >(object); + (void)_this; +} +void ModelProto::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} void ModelProto::SetCachedSize(int size) const { _cached_size_.Set(size); } const ModelProto& ModelProto::default_instance() { - ::google::protobuf::internal::InitSCC(&protobuf_sentencepiece_5fmodel_2eproto::scc_info_ModelProto.base); + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_ModelProto_sentencepiece_5fmodel_2eproto.base); return *internal_default_instance(); } void ModelProto::Clear() { // @@protoc_insertion_point(message_clear_start:sentencepiece.ModelProto) - ::google::protobuf::uint32 cached_has_bits = 0; + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _extensions_.Clear(); pieces_.Clear(); cached_has_bits = _has_bits_[0]; - if (cached_has_bits & 15u) { + if (cached_has_bits & 0x0000000fu) { if (cached_has_bits & 0x00000001u) { - GOOGLE_DCHECK(trainer_spec_ != NULL); + GOOGLE_DCHECK(trainer_spec_ != nullptr); trainer_spec_->Clear(); } if (cached_has_bits & 0x00000002u) { - GOOGLE_DCHECK(normalizer_spec_ != NULL); + GOOGLE_DCHECK(normalizer_spec_ != nullptr); normalizer_spec_->Clear(); } if (cached_has_bits & 0x00000004u) { - GOOGLE_DCHECK(self_test_data_ != NULL); + GOOGLE_DCHECK(self_test_data_ != nullptr); self_test_data_->Clear(); } if (cached_has_bits & 0x00000008u) { - GOOGLE_DCHECK(denormalizer_spec_ != NULL); + GOOGLE_DCHECK(denormalizer_spec_ != nullptr); denormalizer_spec_->Clear(); } } _has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -bool ModelProto::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - ::google::protobuf::internal::LiteUnknownFieldSetter unknown_fields_setter( - &_internal_metadata_); - ::google::protobuf::io::StringOutputStream unknown_fields_output( - unknown_fields_setter.buffer()); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_output, false); - // @@protoc_insertion_point(parse_start:sentencepiece.ModelProto) - for (;;) { - ::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + _internal_metadata_.Clear(); +} + +const char* ModelProto::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { // repeated .sentencepiece.ModelProto.SentencePiece pieces = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, add_pieces())); - } else { - goto handle_unusual; - } - break; - } - + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(_internal_add_pieces(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); + } else goto handle_unusual; + continue; // optional .sentencepiece.TrainerSpec trainer_spec = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, mutable_trainer_spec())); - } else { - goto handle_unusual; - } - break; - } - + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr = ctx->ParseMessage(_internal_mutable_trainer_spec(), ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional .sentencepiece.NormalizerSpec normalizer_spec = 3; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, mutable_normalizer_spec())); - } else { - goto handle_unusual; - } - break; - } - + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { + ptr = ctx->ParseMessage(_internal_mutable_normalizer_spec(), ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional .sentencepiece.SelfTestData self_test_data = 4; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(34u /* 34 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, mutable_self_test_data())); - } else { - goto handle_unusual; - } - break; - } - + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) { + ptr = ctx->ParseMessage(_internal_mutable_self_test_data(), ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; // optional .sentencepiece.NormalizerSpec denormalizer_spec = 5; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(42u /* 42 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, mutable_denormalizer_spec())); - } else { - goto handle_unusual; - } - break; - } - + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { + ptr = ctx->ParseMessage(_internal_mutable_denormalizer_spec(), ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: - if (tag == 0) { + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); goto success; } - if ((1600u <= tag)) { - DO_(_extensions_.ParseField(tag, input, - internal_default_instance(), - &unknown_fields_stream)); - continue; - } - DO_(::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream)); - break; + if ((1600u <= tag)) { + ptr = _extensions_.ParseField(tag, ptr, + internal_default_instance(), &_internal_metadata_, ctx); + CHK_(ptr != nullptr); + continue; } - } - } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while success: - // @@protoc_insertion_point(parse_success:sentencepiece.ModelProto) - return true; + _has_bits_.Or(has_bits); + return ptr; failure: - // @@protoc_insertion_point(parse_failure:sentencepiece.ModelProto) - return false; -#undef DO_ + ptr = nullptr; + goto success; +#undef CHK_ } -void ModelProto::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:sentencepiece.ModelProto) - ::google::protobuf::uint32 cached_has_bits = 0; +::PROTOBUF_NAMESPACE_ID::uint8* ModelProto::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:sentencepiece.ModelProto) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; // repeated .sentencepiece.ModelProto.SentencePiece pieces = 1; for (unsigned int i = 0, - n = static_cast(this->pieces_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 1, - this->pieces(static_cast(i)), - output); + n = static_cast(this->_internal_pieces_size()); i < n; i++) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(1, this->_internal_pieces(i), target, stream); } cached_has_bits = _has_bits_[0]; // optional .sentencepiece.TrainerSpec trainer_spec = 2; if (cached_has_bits & 0x00000001u) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 2, this->_internal_trainer_spec(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage( + 2, _Internal::trainer_spec(this), target, stream); } // optional .sentencepiece.NormalizerSpec normalizer_spec = 3; if (cached_has_bits & 0x00000002u) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 3, this->_internal_normalizer_spec(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage( + 3, _Internal::normalizer_spec(this), target, stream); } // optional .sentencepiece.SelfTestData self_test_data = 4; if (cached_has_bits & 0x00000004u) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 4, this->_internal_self_test_data(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage( + 4, _Internal::self_test_data(this), target, stream); } // optional .sentencepiece.NormalizerSpec denormalizer_spec = 5; if (cached_has_bits & 0x00000008u) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 5, this->_internal_denormalizer_spec(), output); + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage( + 5, _Internal::denormalizer_spec(this), target, stream); } // Extension range [200, 536870912) - _extensions_.SerializeWithCachedSizes( - 200, 536870912, output); + target = _extensions_._InternalSerialize( + 200, 536870912, target, stream); - output->WriteRaw(_internal_metadata_.unknown_fields().data(), - static_cast(_internal_metadata_.unknown_fields().size())); - // @@protoc_insertion_point(serialize_end:sentencepiece.ModelProto) + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:sentencepiece.ModelProto) + return target; } size_t ModelProto::ByteSizeLong() const { @@ -3205,81 +3204,84 @@ size_t ModelProto::ByteSizeLong() const { total_size += _extensions_.ByteSize(); - total_size += _internal_metadata_.unknown_fields().size(); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; // repeated .sentencepiece.ModelProto.SentencePiece pieces = 1; - { - unsigned int count = static_cast(this->pieces_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->pieces(static_cast(i))); - } + total_size += 1UL * this->_internal_pieces_size(); + for (const auto& msg : this->pieces_) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } - if (_has_bits_[0 / 32] & 15u) { + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x0000000fu) { // optional .sentencepiece.TrainerSpec trainer_spec = 2; - if (has_trainer_spec()) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSize( + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *trainer_spec_); } // optional .sentencepiece.NormalizerSpec normalizer_spec = 3; - if (has_normalizer_spec()) { + if (cached_has_bits & 0x00000002u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSize( + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *normalizer_spec_); } // optional .sentencepiece.SelfTestData self_test_data = 4; - if (has_self_test_data()) { + if (cached_has_bits & 0x00000004u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSize( + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *self_test_data_); } // optional .sentencepiece.NormalizerSpec denormalizer_spec = 5; - if (has_denormalizer_spec()) { + if (cached_has_bits & 0x00000008u) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSize( + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *denormalizer_spec_); } } - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; } void ModelProto::CheckTypeAndMergeFrom( - const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); } void ModelProto::MergeFrom(const ModelProto& from) { // @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.ModelProto) GOOGLE_DCHECK_NE(&from, this); _extensions_.MergeFrom(from._extensions_); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; pieces_.MergeFrom(from.pieces_); cached_has_bits = from._has_bits_[0]; - if (cached_has_bits & 15u) { + if (cached_has_bits & 0x0000000fu) { if (cached_has_bits & 0x00000001u) { - mutable_trainer_spec()->::sentencepiece::TrainerSpec::MergeFrom(from.trainer_spec()); + _internal_mutable_trainer_spec()->::sentencepiece::TrainerSpec::MergeFrom(from._internal_trainer_spec()); } if (cached_has_bits & 0x00000002u) { - mutable_normalizer_spec()->::sentencepiece::NormalizerSpec::MergeFrom(from.normalizer_spec()); + _internal_mutable_normalizer_spec()->::sentencepiece::NormalizerSpec::MergeFrom(from._internal_normalizer_spec()); } if (cached_has_bits & 0x00000004u) { - mutable_self_test_data()->::sentencepiece::SelfTestData::MergeFrom(from.self_test_data()); + _internal_mutable_self_test_data()->::sentencepiece::SelfTestData::MergeFrom(from._internal_self_test_data()); } if (cached_has_bits & 0x00000008u) { - mutable_denormalizer_spec()->::sentencepiece::NormalizerSpec::MergeFrom(from.denormalizer_spec()); + _internal_mutable_denormalizer_spec()->::sentencepiece::NormalizerSpec::MergeFrom(from._internal_denormalizer_spec()); } } } @@ -3296,66 +3298,63 @@ bool ModelProto::IsInitialized() const { return false; } - if (!::google::protobuf::internal::AllAreInitialized(this->pieces())) return false; - if (has_trainer_spec()) { - if (!this->trainer_spec_->IsInitialized()) return false; + if (!::PROTOBUF_NAMESPACE_ID::internal::AllAreInitialized(pieces_)) return false; + if (_internal_has_trainer_spec()) { + if (!trainer_spec_->IsInitialized()) return false; } - if (has_normalizer_spec()) { - if (!this->normalizer_spec_->IsInitialized()) return false; + if (_internal_has_normalizer_spec()) { + if (!normalizer_spec_->IsInitialized()) return false; } - if (has_self_test_data()) { - if (!this->self_test_data_->IsInitialized()) return false; + if (_internal_has_self_test_data()) { + if (!self_test_data_->IsInitialized()) return false; } - if (has_denormalizer_spec()) { - if (!this->denormalizer_spec_->IsInitialized()) return false; + if (_internal_has_denormalizer_spec()) { + if (!denormalizer_spec_->IsInitialized()) return false; } return true; } -void ModelProto::Swap(ModelProto* other) { - if (other == this) return; - InternalSwap(other); -} void ModelProto::InternalSwap(ModelProto* other) { using std::swap; - CastToBase(&pieces_)->InternalSwap(CastToBase(&other->pieces_)); - swap(trainer_spec_, other->trainer_spec_); - swap(normalizer_spec_, other->normalizer_spec_); - swap(self_test_data_, other->self_test_data_); - swap(denormalizer_spec_, other->denormalizer_spec_); - swap(_has_bits_[0], other->_has_bits_[0]); - _internal_metadata_.Swap(&other->_internal_metadata_); _extensions_.Swap(&other->_extensions_); + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(_has_bits_[0], other->_has_bits_[0]); + pieces_.InternalSwap(&other->pieces_); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(ModelProto, denormalizer_spec_) + + sizeof(ModelProto::denormalizer_spec_) + - PROTOBUF_FIELD_OFFSET(ModelProto, trainer_spec_)>( + reinterpret_cast(&trainer_spec_), + reinterpret_cast(&other->trainer_spec_)); } -::std::string ModelProto::GetTypeName() const { +std::string ModelProto::GetTypeName() const { return "sentencepiece.ModelProto"; } // @@protoc_insertion_point(namespace_scope) } // namespace sentencepiece -namespace google { -namespace protobuf { -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::TrainerSpec* Arena::CreateMaybeMessage< ::sentencepiece::TrainerSpec >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::TrainerSpec >(arena); +PROTOBUF_NAMESPACE_OPEN +template<> PROTOBUF_NOINLINE ::sentencepiece::TrainerSpec* Arena::CreateMaybeMessage< ::sentencepiece::TrainerSpec >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::TrainerSpec >(arena); } -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::NormalizerSpec* Arena::CreateMaybeMessage< ::sentencepiece::NormalizerSpec >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::NormalizerSpec >(arena); +template<> PROTOBUF_NOINLINE ::sentencepiece::NormalizerSpec* Arena::CreateMaybeMessage< ::sentencepiece::NormalizerSpec >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::NormalizerSpec >(arena); } -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::SelfTestData_Sample* Arena::CreateMaybeMessage< ::sentencepiece::SelfTestData_Sample >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::SelfTestData_Sample >(arena); +template<> PROTOBUF_NOINLINE ::sentencepiece::SelfTestData_Sample* Arena::CreateMaybeMessage< ::sentencepiece::SelfTestData_Sample >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::SelfTestData_Sample >(arena); } -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::SelfTestData* Arena::CreateMaybeMessage< ::sentencepiece::SelfTestData >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::SelfTestData >(arena); +template<> PROTOBUF_NOINLINE ::sentencepiece::SelfTestData* Arena::CreateMaybeMessage< ::sentencepiece::SelfTestData >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::SelfTestData >(arena); } -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::ModelProto_SentencePiece* Arena::CreateMaybeMessage< ::sentencepiece::ModelProto_SentencePiece >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::ModelProto_SentencePiece >(arena); +template<> PROTOBUF_NOINLINE ::sentencepiece::ModelProto_SentencePiece* Arena::CreateMaybeMessage< ::sentencepiece::ModelProto_SentencePiece >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::ModelProto_SentencePiece >(arena); } -template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::sentencepiece::ModelProto* Arena::CreateMaybeMessage< ::sentencepiece::ModelProto >(Arena* arena) { - return Arena::CreateInternal< ::sentencepiece::ModelProto >(arena); +template<> PROTOBUF_NOINLINE ::sentencepiece::ModelProto* Arena::CreateMaybeMessage< ::sentencepiece::ModelProto >(Arena* arena) { + return Arena::CreateMessageInternal< ::sentencepiece::ModelProto >(arena); } -} // namespace protobuf -} // namespace google +PROTOBUF_NAMESPACE_CLOSE // @@protoc_insertion_point(global_scope) +#include diff --git a/src/builtin_pb/sentencepiece_model.pb.h b/src/builtin_pb/sentencepiece_model.pb.h index eeab9cb3..c3a1b0ec 100644 --- a/src/builtin_pb/sentencepiece_model.pb.h +++ b/src/builtin_pb/sentencepiece_model.pb.h @@ -1,24 +1,25 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: sentencepiece_model.proto -#ifndef PROTOBUF_INCLUDED_sentencepiece_5fmodel_2eproto -#define PROTOBUF_INCLUDED_sentencepiece_5fmodel_2eproto +#ifndef GOOGLE_PROTOBUF_INCLUDED_sentencepiece_5fmodel_2eproto +#define GOOGLE_PROTOBUF_INCLUDED_sentencepiece_5fmodel_2eproto +#include #include -#include - -#if GOOGLE_PROTOBUF_VERSION < 3006001 +#include +#if PROTOBUF_VERSION < 3012000 #error This file was generated by a newer version of protoc which is -#error incompatible with your Protocol Buffer headers. Please update +#error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3006001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is -#error incompatible with your Protocol Buffer headers. Please +#error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. #endif +#include #include #include #include @@ -31,19 +32,26 @@ #include // IWYU pragma: export #include // @@protoc_insertion_point(includes) -#define PROTOBUF_INTERNAL_EXPORT_protobuf_sentencepiece_5fmodel_2eproto +#include +#define PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto +PROTOBUF_NAMESPACE_OPEN +namespace internal { +class AnyMetadata; +} // namespace internal +PROTOBUF_NAMESPACE_CLOSE -namespace protobuf_sentencepiece_5fmodel_2eproto { // Internal implementation detail -- do not use these members. -struct TableStruct { - static const ::google::protobuf::internal::ParseTableField entries[]; - static const ::google::protobuf::internal::AuxillaryParseTableField aux[]; - static const ::google::protobuf::internal::ParseTable schema[6]; - static const ::google::protobuf::internal::FieldMetadata field_metadata[]; - static const ::google::protobuf::internal::SerializationTable serialization_table[]; - static const ::google::protobuf::uint32 offsets[]; +struct TableStruct_sentencepiece_5fmodel_2eproto { + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[6] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; + static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; + static const ::PROTOBUF_NAMESPACE_ID::uint32 offsets[]; }; -} // namespace protobuf_sentencepiece_5fmodel_2eproto namespace sentencepiece { class ModelProto; class ModelProtoDefaultTypeInternal; @@ -64,30 +72,38 @@ class TrainerSpec; class TrainerSpecDefaultTypeInternal; extern TrainerSpecDefaultTypeInternal _TrainerSpec_default_instance_; } // namespace sentencepiece -namespace google { -namespace protobuf { +PROTOBUF_NAMESPACE_OPEN template<> ::sentencepiece::ModelProto* Arena::CreateMaybeMessage<::sentencepiece::ModelProto>(Arena*); template<> ::sentencepiece::ModelProto_SentencePiece* Arena::CreateMaybeMessage<::sentencepiece::ModelProto_SentencePiece>(Arena*); template<> ::sentencepiece::NormalizerSpec* Arena::CreateMaybeMessage<::sentencepiece::NormalizerSpec>(Arena*); template<> ::sentencepiece::SelfTestData* Arena::CreateMaybeMessage<::sentencepiece::SelfTestData>(Arena*); template<> ::sentencepiece::SelfTestData_Sample* Arena::CreateMaybeMessage<::sentencepiece::SelfTestData_Sample>(Arena*); template<> ::sentencepiece::TrainerSpec* Arena::CreateMaybeMessage<::sentencepiece::TrainerSpec>(Arena*); -} // namespace protobuf -} // namespace google +PROTOBUF_NAMESPACE_CLOSE namespace sentencepiece { -enum TrainerSpec_ModelType { +enum TrainerSpec_ModelType : int { TrainerSpec_ModelType_UNIGRAM = 1, TrainerSpec_ModelType_BPE = 2, TrainerSpec_ModelType_WORD = 3, TrainerSpec_ModelType_CHAR = 4 }; bool TrainerSpec_ModelType_IsValid(int value); -const TrainerSpec_ModelType TrainerSpec_ModelType_ModelType_MIN = TrainerSpec_ModelType_UNIGRAM; -const TrainerSpec_ModelType TrainerSpec_ModelType_ModelType_MAX = TrainerSpec_ModelType_CHAR; -const int TrainerSpec_ModelType_ModelType_ARRAYSIZE = TrainerSpec_ModelType_ModelType_MAX + 1; - -enum ModelProto_SentencePiece_Type { +constexpr TrainerSpec_ModelType TrainerSpec_ModelType_ModelType_MIN = TrainerSpec_ModelType_UNIGRAM; +constexpr TrainerSpec_ModelType TrainerSpec_ModelType_ModelType_MAX = TrainerSpec_ModelType_CHAR; +constexpr int TrainerSpec_ModelType_ModelType_ARRAYSIZE = TrainerSpec_ModelType_ModelType_MAX + 1; + +const std::string& TrainerSpec_ModelType_Name(TrainerSpec_ModelType value); +template +inline const std::string& TrainerSpec_ModelType_Name(T enum_t_value) { + static_assert(::std::is_same::value || + ::std::is_integral::value, + "Incorrect type passed to function TrainerSpec_ModelType_Name."); + return TrainerSpec_ModelType_Name(static_cast(enum_t_value)); +} +bool TrainerSpec_ModelType_Parse( + const std::string& name, TrainerSpec_ModelType* value); +enum ModelProto_SentencePiece_Type : int { ModelProto_SentencePiece_Type_NORMAL = 1, ModelProto_SentencePiece_Type_UNKNOWN = 2, ModelProto_SentencePiece_Type_CONTROL = 3, @@ -96,43 +112,52 @@ enum ModelProto_SentencePiece_Type { ModelProto_SentencePiece_Type_UNUSED = 5 }; bool ModelProto_SentencePiece_Type_IsValid(int value); -const ModelProto_SentencePiece_Type ModelProto_SentencePiece_Type_Type_MIN = ModelProto_SentencePiece_Type_NORMAL; -const ModelProto_SentencePiece_Type ModelProto_SentencePiece_Type_Type_MAX = ModelProto_SentencePiece_Type_BYTE; -const int ModelProto_SentencePiece_Type_Type_ARRAYSIZE = ModelProto_SentencePiece_Type_Type_MAX + 1; - +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece_Type_Type_MIN = ModelProto_SentencePiece_Type_NORMAL; +constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece_Type_Type_MAX = ModelProto_SentencePiece_Type_BYTE; +constexpr int ModelProto_SentencePiece_Type_Type_ARRAYSIZE = ModelProto_SentencePiece_Type_Type_MAX + 1; + +const std::string& ModelProto_SentencePiece_Type_Name(ModelProto_SentencePiece_Type value); +template +inline const std::string& ModelProto_SentencePiece_Type_Name(T enum_t_value) { + static_assert(::std::is_same::value || + ::std::is_integral::value, + "Incorrect type passed to function ModelProto_SentencePiece_Type_Name."); + return ModelProto_SentencePiece_Type_Name(static_cast(enum_t_value)); +} +bool ModelProto_SentencePiece_Type_Parse( + const std::string& name, ModelProto_SentencePiece_Type* value); // =================================================================== -class TrainerSpec : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.TrainerSpec) */ { +class TrainerSpec PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.TrainerSpec) */ { public: - TrainerSpec(); + inline TrainerSpec() : TrainerSpec(nullptr) {}; virtual ~TrainerSpec(); TrainerSpec(const TrainerSpec& from); - - inline TrainerSpec& operator=(const TrainerSpec& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 TrainerSpec(TrainerSpec&& from) noexcept : TrainerSpec() { *this = ::std::move(from); } + inline TrainerSpec& operator=(const TrainerSpec& from) { + CopyFrom(from); + return *this; + } inline TrainerSpec& operator=(TrainerSpec&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const TrainerSpec& default_instance(); @@ -145,649 +170,933 @@ class TrainerSpec : public ::google::protobuf::MessageLite /* @@protoc_insertion static constexpr int kIndexInFileMessages = 0; - void Swap(TrainerSpec* other); friend void swap(TrainerSpec& a, TrainerSpec& b) { a.Swap(&b); } + inline void Swap(TrainerSpec* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(TrainerSpec* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline TrainerSpec* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - TrainerSpec* New(::google::protobuf::Arena* arena) const final { + TrainerSpec* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const TrainerSpec& from); void MergeFrom(const TrainerSpec& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(TrainerSpec* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.TrainerSpec"; } + protected: + explicit TrainerSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- typedef TrainerSpec_ModelType ModelType; - static const ModelType UNIGRAM = + static constexpr ModelType UNIGRAM = TrainerSpec_ModelType_UNIGRAM; - static const ModelType BPE = + static constexpr ModelType BPE = TrainerSpec_ModelType_BPE; - static const ModelType WORD = + static constexpr ModelType WORD = TrainerSpec_ModelType_WORD; - static const ModelType CHAR = + static constexpr ModelType CHAR = TrainerSpec_ModelType_CHAR; static inline bool ModelType_IsValid(int value) { return TrainerSpec_ModelType_IsValid(value); } - static const ModelType ModelType_MIN = + static constexpr ModelType ModelType_MIN = TrainerSpec_ModelType_ModelType_MIN; - static const ModelType ModelType_MAX = + static constexpr ModelType ModelType_MAX = TrainerSpec_ModelType_ModelType_MAX; - static const int ModelType_ARRAYSIZE = + static constexpr int ModelType_ARRAYSIZE = TrainerSpec_ModelType_ModelType_ARRAYSIZE; + template + static inline const std::string& ModelType_Name(T enum_t_value) { + static_assert(::std::is_same::value || + ::std::is_integral::value, + "Incorrect type passed to function ModelType_Name."); + return TrainerSpec_ModelType_Name(enum_t_value); + } + static inline bool ModelType_Parse(const std::string& name, + ModelType* value) { + return TrainerSpec_ModelType_Parse(name, value); + } // accessors ------------------------------------------------------- + enum : int { + kInputFieldNumber = 1, + kAcceptLanguageFieldNumber = 5, + kControlSymbolsFieldNumber = 30, + kUserDefinedSymbolsFieldNumber = 31, + kModelPrefixFieldNumber = 2, + kInputFormatFieldNumber = 7, + kRequiredCharsFieldNumber = 36, + kUnkSurfaceFieldNumber = 44, + kUnkPieceFieldNumber = 45, + kBosPieceFieldNumber = 46, + kEosPieceFieldNumber = 47, + kPadPieceFieldNumber = 48, + kSelfTestSampleSizeFieldNumber = 6, + kInputSentenceSizeFieldNumber = 11, + kMiningSentenceSizeFieldNumber = 12, + kTrainingSentenceSizeFieldNumber = 13, + kTreatWhitespaceAsSuffixFieldNumber = 24, + kSplitDigitsFieldNumber = 25, + kByteFallbackFieldNumber = 35, + kUseAllVocabFieldNumber = 34, + kUnkIdFieldNumber = 40, + kTrainExtremelyLargeCorpusFieldNumber = 49, + kModelTypeFieldNumber = 3, + kVocabSizeFieldNumber = 4, + kCharacterCoverageFieldNumber = 10, + kSeedSentencepieceSizeFieldNumber = 14, + kShrinkingFactorFieldNumber = 15, + kNumThreadsFieldNumber = 16, + kNumSubIterationsFieldNumber = 17, + kMaxSentenceLengthFieldNumber = 18, + kMaxSentencepieceLengthFieldNumber = 20, + kShuffleInputSentenceFieldNumber = 19, + kSplitByUnicodeScriptFieldNumber = 21, + kSplitByNumberFieldNumber = 23, + kSplitByWhitespaceFieldNumber = 22, + kVocabularyOutputPieceScoreFieldNumber = 32, + kHardVocabLimitFieldNumber = 33, + kBosIdFieldNumber = 41, + kEosIdFieldNumber = 42, + kPadIdFieldNumber = 43, + }; // repeated string input = 1; int input_size() const; + private: + int _internal_input_size() const; + public: void clear_input(); - static const int kInputFieldNumber = 1; - const ::std::string& input(int index) const; - ::std::string* mutable_input(int index); - void set_input(int index, const ::std::string& value); - #if LANG_CXX11 - void set_input(int index, ::std::string&& value); - #endif + const std::string& input(int index) const; + std::string* mutable_input(int index); + void set_input(int index, const std::string& value); + void set_input(int index, std::string&& value); void set_input(int index, const char* value); void set_input(int index, const char* value, size_t size); - ::std::string* add_input(); - void add_input(const ::std::string& value); - #if LANG_CXX11 - void add_input(::std::string&& value); - #endif + std::string* add_input(); + void add_input(const std::string& value); + void add_input(std::string&& value); void add_input(const char* value); void add_input(const char* value, size_t size); - const ::google::protobuf::RepeatedPtrField< ::std::string>& input() const; - ::google::protobuf::RepeatedPtrField< ::std::string>* mutable_input(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& input() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_input(); + private: + const std::string& _internal_input(int index) const; + std::string* _internal_add_input(); + public: // repeated string accept_language = 5; int accept_language_size() const; + private: + int _internal_accept_language_size() const; + public: void clear_accept_language(); - static const int kAcceptLanguageFieldNumber = 5; - const ::std::string& accept_language(int index) const; - ::std::string* mutable_accept_language(int index); - void set_accept_language(int index, const ::std::string& value); - #if LANG_CXX11 - void set_accept_language(int index, ::std::string&& value); - #endif + const std::string& accept_language(int index) const; + std::string* mutable_accept_language(int index); + void set_accept_language(int index, const std::string& value); + void set_accept_language(int index, std::string&& value); void set_accept_language(int index, const char* value); void set_accept_language(int index, const char* value, size_t size); - ::std::string* add_accept_language(); - void add_accept_language(const ::std::string& value); - #if LANG_CXX11 - void add_accept_language(::std::string&& value); - #endif + std::string* add_accept_language(); + void add_accept_language(const std::string& value); + void add_accept_language(std::string&& value); void add_accept_language(const char* value); void add_accept_language(const char* value, size_t size); - const ::google::protobuf::RepeatedPtrField< ::std::string>& accept_language() const; - ::google::protobuf::RepeatedPtrField< ::std::string>* mutable_accept_language(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& accept_language() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_accept_language(); + private: + const std::string& _internal_accept_language(int index) const; + std::string* _internal_add_accept_language(); + public: // repeated string control_symbols = 30; int control_symbols_size() const; + private: + int _internal_control_symbols_size() const; + public: void clear_control_symbols(); - static const int kControlSymbolsFieldNumber = 30; - const ::std::string& control_symbols(int index) const; - ::std::string* mutable_control_symbols(int index); - void set_control_symbols(int index, const ::std::string& value); - #if LANG_CXX11 - void set_control_symbols(int index, ::std::string&& value); - #endif + const std::string& control_symbols(int index) const; + std::string* mutable_control_symbols(int index); + void set_control_symbols(int index, const std::string& value); + void set_control_symbols(int index, std::string&& value); void set_control_symbols(int index, const char* value); void set_control_symbols(int index, const char* value, size_t size); - ::std::string* add_control_symbols(); - void add_control_symbols(const ::std::string& value); - #if LANG_CXX11 - void add_control_symbols(::std::string&& value); - #endif + std::string* add_control_symbols(); + void add_control_symbols(const std::string& value); + void add_control_symbols(std::string&& value); void add_control_symbols(const char* value); void add_control_symbols(const char* value, size_t size); - const ::google::protobuf::RepeatedPtrField< ::std::string>& control_symbols() const; - ::google::protobuf::RepeatedPtrField< ::std::string>* mutable_control_symbols(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& control_symbols() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_control_symbols(); + private: + const std::string& _internal_control_symbols(int index) const; + std::string* _internal_add_control_symbols(); + public: // repeated string user_defined_symbols = 31; int user_defined_symbols_size() const; + private: + int _internal_user_defined_symbols_size() const; + public: void clear_user_defined_symbols(); - static const int kUserDefinedSymbolsFieldNumber = 31; - const ::std::string& user_defined_symbols(int index) const; - ::std::string* mutable_user_defined_symbols(int index); - void set_user_defined_symbols(int index, const ::std::string& value); - #if LANG_CXX11 - void set_user_defined_symbols(int index, ::std::string&& value); - #endif + const std::string& user_defined_symbols(int index) const; + std::string* mutable_user_defined_symbols(int index); + void set_user_defined_symbols(int index, const std::string& value); + void set_user_defined_symbols(int index, std::string&& value); void set_user_defined_symbols(int index, const char* value); void set_user_defined_symbols(int index, const char* value, size_t size); - ::std::string* add_user_defined_symbols(); - void add_user_defined_symbols(const ::std::string& value); - #if LANG_CXX11 - void add_user_defined_symbols(::std::string&& value); - #endif + std::string* add_user_defined_symbols(); + void add_user_defined_symbols(const std::string& value); + void add_user_defined_symbols(std::string&& value); void add_user_defined_symbols(const char* value); void add_user_defined_symbols(const char* value, size_t size); - const ::google::protobuf::RepeatedPtrField< ::std::string>& user_defined_symbols() const; - ::google::protobuf::RepeatedPtrField< ::std::string>* mutable_user_defined_symbols(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& user_defined_symbols() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_user_defined_symbols(); + private: + const std::string& _internal_user_defined_symbols(int index) const; + std::string* _internal_add_user_defined_symbols(); + public: // optional string model_prefix = 2; bool has_model_prefix() const; + private: + bool _internal_has_model_prefix() const; + public: void clear_model_prefix(); - static const int kModelPrefixFieldNumber = 2; - const ::std::string& model_prefix() const; - void set_model_prefix(const ::std::string& value); - #if LANG_CXX11 - void set_model_prefix(::std::string&& value); - #endif + const std::string& model_prefix() const; + void set_model_prefix(const std::string& value); + void set_model_prefix(std::string&& value); void set_model_prefix(const char* value); void set_model_prefix(const char* value, size_t size); - ::std::string* mutable_model_prefix(); - ::std::string* release_model_prefix(); - void set_allocated_model_prefix(::std::string* model_prefix); + std::string* mutable_model_prefix(); + std::string* release_model_prefix(); + void set_allocated_model_prefix(std::string* model_prefix); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_model_prefix(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_model_prefix( + std::string* model_prefix); + private: + const std::string& _internal_model_prefix() const; + void _internal_set_model_prefix(const std::string& value); + std::string* _internal_mutable_model_prefix(); + public: // optional string input_format = 7; bool has_input_format() const; + private: + bool _internal_has_input_format() const; + public: void clear_input_format(); - static const int kInputFormatFieldNumber = 7; - const ::std::string& input_format() const; - void set_input_format(const ::std::string& value); - #if LANG_CXX11 - void set_input_format(::std::string&& value); - #endif + const std::string& input_format() const; + void set_input_format(const std::string& value); + void set_input_format(std::string&& value); void set_input_format(const char* value); void set_input_format(const char* value, size_t size); - ::std::string* mutable_input_format(); - ::std::string* release_input_format(); - void set_allocated_input_format(::std::string* input_format); + std::string* mutable_input_format(); + std::string* release_input_format(); + void set_allocated_input_format(std::string* input_format); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_input_format(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_input_format( + std::string* input_format); + private: + const std::string& _internal_input_format() const; + void _internal_set_input_format(const std::string& value); + std::string* _internal_mutable_input_format(); + public: // optional string required_chars = 36; bool has_required_chars() const; + private: + bool _internal_has_required_chars() const; + public: void clear_required_chars(); - static const int kRequiredCharsFieldNumber = 36; - const ::std::string& required_chars() const; - void set_required_chars(const ::std::string& value); - #if LANG_CXX11 - void set_required_chars(::std::string&& value); - #endif + const std::string& required_chars() const; + void set_required_chars(const std::string& value); + void set_required_chars(std::string&& value); void set_required_chars(const char* value); void set_required_chars(const char* value, size_t size); - ::std::string* mutable_required_chars(); - ::std::string* release_required_chars(); - void set_allocated_required_chars(::std::string* required_chars); + std::string* mutable_required_chars(); + std::string* release_required_chars(); + void set_allocated_required_chars(std::string* required_chars); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_required_chars(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_required_chars( + std::string* required_chars); + private: + const std::string& _internal_required_chars() const; + void _internal_set_required_chars(const std::string& value); + std::string* _internal_mutable_required_chars(); + public: // optional string unk_surface = 44 [default = " \342\201\207 "]; bool has_unk_surface() const; + private: + bool _internal_has_unk_surface() const; + public: void clear_unk_surface(); - static const int kUnkSurfaceFieldNumber = 44; - const ::std::string& unk_surface() const; - void set_unk_surface(const ::std::string& value); - #if LANG_CXX11 - void set_unk_surface(::std::string&& value); - #endif + const std::string& unk_surface() const; + void set_unk_surface(const std::string& value); + void set_unk_surface(std::string&& value); void set_unk_surface(const char* value); void set_unk_surface(const char* value, size_t size); - ::std::string* mutable_unk_surface(); - ::std::string* release_unk_surface(); - void set_allocated_unk_surface(::std::string* unk_surface); + std::string* mutable_unk_surface(); + std::string* release_unk_surface(); + void set_allocated_unk_surface(std::string* unk_surface); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_unk_surface(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_unk_surface( + std::string* unk_surface); + private: + const std::string& _internal_unk_surface() const; + void _internal_set_unk_surface(const std::string& value); + std::string* _internal_mutable_unk_surface(); + public: // optional string unk_piece = 45 [default = ""]; bool has_unk_piece() const; + private: + bool _internal_has_unk_piece() const; + public: void clear_unk_piece(); - static const int kUnkPieceFieldNumber = 45; - const ::std::string& unk_piece() const; - void set_unk_piece(const ::std::string& value); - #if LANG_CXX11 - void set_unk_piece(::std::string&& value); - #endif + const std::string& unk_piece() const; + void set_unk_piece(const std::string& value); + void set_unk_piece(std::string&& value); void set_unk_piece(const char* value); void set_unk_piece(const char* value, size_t size); - ::std::string* mutable_unk_piece(); - ::std::string* release_unk_piece(); - void set_allocated_unk_piece(::std::string* unk_piece); + std::string* mutable_unk_piece(); + std::string* release_unk_piece(); + void set_allocated_unk_piece(std::string* unk_piece); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_unk_piece(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_unk_piece( + std::string* unk_piece); + private: + const std::string& _internal_unk_piece() const; + void _internal_set_unk_piece(const std::string& value); + std::string* _internal_mutable_unk_piece(); + public: // optional string bos_piece = 46 [default = ""]; bool has_bos_piece() const; + private: + bool _internal_has_bos_piece() const; + public: void clear_bos_piece(); - static const int kBosPieceFieldNumber = 46; - const ::std::string& bos_piece() const; - void set_bos_piece(const ::std::string& value); - #if LANG_CXX11 - void set_bos_piece(::std::string&& value); - #endif + const std::string& bos_piece() const; + void set_bos_piece(const std::string& value); + void set_bos_piece(std::string&& value); void set_bos_piece(const char* value); void set_bos_piece(const char* value, size_t size); - ::std::string* mutable_bos_piece(); - ::std::string* release_bos_piece(); - void set_allocated_bos_piece(::std::string* bos_piece); + std::string* mutable_bos_piece(); + std::string* release_bos_piece(); + void set_allocated_bos_piece(std::string* bos_piece); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_bos_piece(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_bos_piece( + std::string* bos_piece); + private: + const std::string& _internal_bos_piece() const; + void _internal_set_bos_piece(const std::string& value); + std::string* _internal_mutable_bos_piece(); + public: // optional string eos_piece = 47 [default = ""]; bool has_eos_piece() const; + private: + bool _internal_has_eos_piece() const; + public: void clear_eos_piece(); - static const int kEosPieceFieldNumber = 47; - const ::std::string& eos_piece() const; - void set_eos_piece(const ::std::string& value); - #if LANG_CXX11 - void set_eos_piece(::std::string&& value); - #endif + const std::string& eos_piece() const; + void set_eos_piece(const std::string& value); + void set_eos_piece(std::string&& value); void set_eos_piece(const char* value); void set_eos_piece(const char* value, size_t size); - ::std::string* mutable_eos_piece(); - ::std::string* release_eos_piece(); - void set_allocated_eos_piece(::std::string* eos_piece); + std::string* mutable_eos_piece(); + std::string* release_eos_piece(); + void set_allocated_eos_piece(std::string* eos_piece); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_eos_piece(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_eos_piece( + std::string* eos_piece); + private: + const std::string& _internal_eos_piece() const; + void _internal_set_eos_piece(const std::string& value); + std::string* _internal_mutable_eos_piece(); + public: // optional string pad_piece = 48 [default = ""]; bool has_pad_piece() const; + private: + bool _internal_has_pad_piece() const; + public: void clear_pad_piece(); - static const int kPadPieceFieldNumber = 48; - const ::std::string& pad_piece() const; - void set_pad_piece(const ::std::string& value); - #if LANG_CXX11 - void set_pad_piece(::std::string&& value); - #endif + const std::string& pad_piece() const; + void set_pad_piece(const std::string& value); + void set_pad_piece(std::string&& value); void set_pad_piece(const char* value); void set_pad_piece(const char* value, size_t size); - ::std::string* mutable_pad_piece(); - ::std::string* release_pad_piece(); - void set_allocated_pad_piece(::std::string* pad_piece); + std::string* mutable_pad_piece(); + std::string* release_pad_piece(); + void set_allocated_pad_piece(std::string* pad_piece); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_pad_piece(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_pad_piece( + std::string* pad_piece); + private: + const std::string& _internal_pad_piece() const; + void _internal_set_pad_piece(const std::string& value); + std::string* _internal_mutable_pad_piece(); + public: // optional int32 self_test_sample_size = 6 [default = 0]; bool has_self_test_sample_size() const; + private: + bool _internal_has_self_test_sample_size() const; + public: void clear_self_test_sample_size(); - static const int kSelfTestSampleSizeFieldNumber = 6; - ::google::protobuf::int32 self_test_sample_size() const; - void set_self_test_sample_size(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 self_test_sample_size() const; + void set_self_test_sample_size(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_self_test_sample_size() const; + void _internal_set_self_test_sample_size(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 input_sentence_size = 11 [default = 0]; bool has_input_sentence_size() const; + private: + bool _internal_has_input_sentence_size() const; + public: void clear_input_sentence_size(); - static const int kInputSentenceSizeFieldNumber = 11; - ::google::protobuf::int32 input_sentence_size() const; - void set_input_sentence_size(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 input_sentence_size() const; + void set_input_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_input_sentence_size() const; + void _internal_set_input_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 mining_sentence_size = 12 [deprecated = true]; - GOOGLE_PROTOBUF_DEPRECATED_ATTR bool has_mining_sentence_size() const; - GOOGLE_PROTOBUF_DEPRECATED_ATTR void clear_mining_sentence_size(); - GOOGLE_PROTOBUF_DEPRECATED_ATTR static const int kMiningSentenceSizeFieldNumber = 12; - GOOGLE_PROTOBUF_DEPRECATED_ATTR ::google::protobuf::int32 mining_sentence_size() const; - GOOGLE_PROTOBUF_DEPRECATED_ATTR void set_mining_sentence_size(::google::protobuf::int32 value); + PROTOBUF_DEPRECATED bool has_mining_sentence_size() const; + private: + bool _internal_has_mining_sentence_size() const; + public: + PROTOBUF_DEPRECATED void clear_mining_sentence_size(); + PROTOBUF_DEPRECATED ::PROTOBUF_NAMESPACE_ID::int32 mining_sentence_size() const; + PROTOBUF_DEPRECATED void set_mining_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_mining_sentence_size() const; + void _internal_set_mining_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 training_sentence_size = 13 [deprecated = true]; - GOOGLE_PROTOBUF_DEPRECATED_ATTR bool has_training_sentence_size() const; - GOOGLE_PROTOBUF_DEPRECATED_ATTR void clear_training_sentence_size(); - GOOGLE_PROTOBUF_DEPRECATED_ATTR static const int kTrainingSentenceSizeFieldNumber = 13; - GOOGLE_PROTOBUF_DEPRECATED_ATTR ::google::protobuf::int32 training_sentence_size() const; - GOOGLE_PROTOBUF_DEPRECATED_ATTR void set_training_sentence_size(::google::protobuf::int32 value); + PROTOBUF_DEPRECATED bool has_training_sentence_size() const; + private: + bool _internal_has_training_sentence_size() const; + public: + PROTOBUF_DEPRECATED void clear_training_sentence_size(); + PROTOBUF_DEPRECATED ::PROTOBUF_NAMESPACE_ID::int32 training_sentence_size() const; + PROTOBUF_DEPRECATED void set_training_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_training_sentence_size() const; + void _internal_set_training_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional bool treat_whitespace_as_suffix = 24 [default = false]; bool has_treat_whitespace_as_suffix() const; + private: + bool _internal_has_treat_whitespace_as_suffix() const; + public: void clear_treat_whitespace_as_suffix(); - static const int kTreatWhitespaceAsSuffixFieldNumber = 24; bool treat_whitespace_as_suffix() const; void set_treat_whitespace_as_suffix(bool value); + private: + bool _internal_treat_whitespace_as_suffix() const; + void _internal_set_treat_whitespace_as_suffix(bool value); + public: // optional bool split_digits = 25 [default = false]; bool has_split_digits() const; + private: + bool _internal_has_split_digits() const; + public: void clear_split_digits(); - static const int kSplitDigitsFieldNumber = 25; bool split_digits() const; void set_split_digits(bool value); + private: + bool _internal_split_digits() const; + void _internal_set_split_digits(bool value); + public: // optional bool byte_fallback = 35 [default = false]; bool has_byte_fallback() const; + private: + bool _internal_has_byte_fallback() const; + public: void clear_byte_fallback(); - static const int kByteFallbackFieldNumber = 35; bool byte_fallback() const; void set_byte_fallback(bool value); + private: + bool _internal_byte_fallback() const; + void _internal_set_byte_fallback(bool value); + public: // optional bool use_all_vocab = 34 [default = false]; bool has_use_all_vocab() const; + private: + bool _internal_has_use_all_vocab() const; + public: void clear_use_all_vocab(); - static const int kUseAllVocabFieldNumber = 34; bool use_all_vocab() const; void set_use_all_vocab(bool value); + private: + bool _internal_use_all_vocab() const; + void _internal_set_use_all_vocab(bool value); + public: // optional int32 unk_id = 40 [default = 0]; bool has_unk_id() const; + private: + bool _internal_has_unk_id() const; + public: void clear_unk_id(); - static const int kUnkIdFieldNumber = 40; - ::google::protobuf::int32 unk_id() const; - void set_unk_id(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 unk_id() const; + void set_unk_id(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_unk_id() const; + void _internal_set_unk_id(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional bool train_extremely_large_corpus = 49 [default = false]; bool has_train_extremely_large_corpus() const; + private: + bool _internal_has_train_extremely_large_corpus() const; + public: void clear_train_extremely_large_corpus(); - static const int kTrainExtremelyLargeCorpusFieldNumber = 49; bool train_extremely_large_corpus() const; void set_train_extremely_large_corpus(bool value); + private: + bool _internal_train_extremely_large_corpus() const; + void _internal_set_train_extremely_large_corpus(bool value); + public: // optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default = UNIGRAM]; bool has_model_type() const; + private: + bool _internal_has_model_type() const; + public: void clear_model_type(); - static const int kModelTypeFieldNumber = 3; ::sentencepiece::TrainerSpec_ModelType model_type() const; void set_model_type(::sentencepiece::TrainerSpec_ModelType value); + private: + ::sentencepiece::TrainerSpec_ModelType _internal_model_type() const; + void _internal_set_model_type(::sentencepiece::TrainerSpec_ModelType value); + public: // optional int32 vocab_size = 4 [default = 8000]; bool has_vocab_size() const; + private: + bool _internal_has_vocab_size() const; + public: void clear_vocab_size(); - static const int kVocabSizeFieldNumber = 4; - ::google::protobuf::int32 vocab_size() const; - void set_vocab_size(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 vocab_size() const; + void set_vocab_size(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_vocab_size() const; + void _internal_set_vocab_size(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional float character_coverage = 10 [default = 0.9995]; bool has_character_coverage() const; + private: + bool _internal_has_character_coverage() const; + public: void clear_character_coverage(); - static const int kCharacterCoverageFieldNumber = 10; float character_coverage() const; void set_character_coverage(float value); + private: + float _internal_character_coverage() const; + void _internal_set_character_coverage(float value); + public: // optional int32 seed_sentencepiece_size = 14 [default = 1000000]; bool has_seed_sentencepiece_size() const; + private: + bool _internal_has_seed_sentencepiece_size() const; + public: void clear_seed_sentencepiece_size(); - static const int kSeedSentencepieceSizeFieldNumber = 14; - ::google::protobuf::int32 seed_sentencepiece_size() const; - void set_seed_sentencepiece_size(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 seed_sentencepiece_size() const; + void set_seed_sentencepiece_size(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_seed_sentencepiece_size() const; + void _internal_set_seed_sentencepiece_size(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional float shrinking_factor = 15 [default = 0.75]; bool has_shrinking_factor() const; + private: + bool _internal_has_shrinking_factor() const; + public: void clear_shrinking_factor(); - static const int kShrinkingFactorFieldNumber = 15; float shrinking_factor() const; void set_shrinking_factor(float value); + private: + float _internal_shrinking_factor() const; + void _internal_set_shrinking_factor(float value); + public: // optional int32 num_threads = 16 [default = 16]; bool has_num_threads() const; + private: + bool _internal_has_num_threads() const; + public: void clear_num_threads(); - static const int kNumThreadsFieldNumber = 16; - ::google::protobuf::int32 num_threads() const; - void set_num_threads(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 num_threads() const; + void set_num_threads(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_num_threads() const; + void _internal_set_num_threads(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 num_sub_iterations = 17 [default = 2]; bool has_num_sub_iterations() const; + private: + bool _internal_has_num_sub_iterations() const; + public: void clear_num_sub_iterations(); - static const int kNumSubIterationsFieldNumber = 17; - ::google::protobuf::int32 num_sub_iterations() const; - void set_num_sub_iterations(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 num_sub_iterations() const; + void set_num_sub_iterations(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_num_sub_iterations() const; + void _internal_set_num_sub_iterations(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 max_sentence_length = 18 [default = 4192]; bool has_max_sentence_length() const; + private: + bool _internal_has_max_sentence_length() const; + public: void clear_max_sentence_length(); - static const int kMaxSentenceLengthFieldNumber = 18; - ::google::protobuf::int32 max_sentence_length() const; - void set_max_sentence_length(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 max_sentence_length() const; + void set_max_sentence_length(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_max_sentence_length() const; + void _internal_set_max_sentence_length(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 max_sentencepiece_length = 20 [default = 16]; bool has_max_sentencepiece_length() const; + private: + bool _internal_has_max_sentencepiece_length() const; + public: void clear_max_sentencepiece_length(); - static const int kMaxSentencepieceLengthFieldNumber = 20; - ::google::protobuf::int32 max_sentencepiece_length() const; - void set_max_sentencepiece_length(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 max_sentencepiece_length() const; + void set_max_sentencepiece_length(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_max_sentencepiece_length() const; + void _internal_set_max_sentencepiece_length(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional bool shuffle_input_sentence = 19 [default = true]; bool has_shuffle_input_sentence() const; + private: + bool _internal_has_shuffle_input_sentence() const; + public: void clear_shuffle_input_sentence(); - static const int kShuffleInputSentenceFieldNumber = 19; bool shuffle_input_sentence() const; void set_shuffle_input_sentence(bool value); + private: + bool _internal_shuffle_input_sentence() const; + void _internal_set_shuffle_input_sentence(bool value); + public: // optional bool split_by_unicode_script = 21 [default = true]; bool has_split_by_unicode_script() const; + private: + bool _internal_has_split_by_unicode_script() const; + public: void clear_split_by_unicode_script(); - static const int kSplitByUnicodeScriptFieldNumber = 21; bool split_by_unicode_script() const; void set_split_by_unicode_script(bool value); + private: + bool _internal_split_by_unicode_script() const; + void _internal_set_split_by_unicode_script(bool value); + public: // optional bool split_by_number = 23 [default = true]; bool has_split_by_number() const; + private: + bool _internal_has_split_by_number() const; + public: void clear_split_by_number(); - static const int kSplitByNumberFieldNumber = 23; bool split_by_number() const; void set_split_by_number(bool value); + private: + bool _internal_split_by_number() const; + void _internal_set_split_by_number(bool value); + public: // optional bool split_by_whitespace = 22 [default = true]; bool has_split_by_whitespace() const; + private: + bool _internal_has_split_by_whitespace() const; + public: void clear_split_by_whitespace(); - static const int kSplitByWhitespaceFieldNumber = 22; bool split_by_whitespace() const; void set_split_by_whitespace(bool value); + private: + bool _internal_split_by_whitespace() const; + void _internal_set_split_by_whitespace(bool value); + public: // optional bool vocabulary_output_piece_score = 32 [default = true]; bool has_vocabulary_output_piece_score() const; + private: + bool _internal_has_vocabulary_output_piece_score() const; + public: void clear_vocabulary_output_piece_score(); - static const int kVocabularyOutputPieceScoreFieldNumber = 32; bool vocabulary_output_piece_score() const; void set_vocabulary_output_piece_score(bool value); + private: + bool _internal_vocabulary_output_piece_score() const; + void _internal_set_vocabulary_output_piece_score(bool value); + public: // optional bool hard_vocab_limit = 33 [default = true]; bool has_hard_vocab_limit() const; + private: + bool _internal_has_hard_vocab_limit() const; + public: void clear_hard_vocab_limit(); - static const int kHardVocabLimitFieldNumber = 33; bool hard_vocab_limit() const; void set_hard_vocab_limit(bool value); + private: + bool _internal_hard_vocab_limit() const; + void _internal_set_hard_vocab_limit(bool value); + public: // optional int32 bos_id = 41 [default = 1]; bool has_bos_id() const; + private: + bool _internal_has_bos_id() const; + public: void clear_bos_id(); - static const int kBosIdFieldNumber = 41; - ::google::protobuf::int32 bos_id() const; - void set_bos_id(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 bos_id() const; + void set_bos_id(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_bos_id() const; + void _internal_set_bos_id(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 eos_id = 42 [default = 2]; bool has_eos_id() const; + private: + bool _internal_has_eos_id() const; + public: void clear_eos_id(); - static const int kEosIdFieldNumber = 42; - ::google::protobuf::int32 eos_id() const; - void set_eos_id(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 eos_id() const; + void set_eos_id(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_eos_id() const; + void _internal_set_eos_id(::PROTOBUF_NAMESPACE_ID::int32 value); + public: // optional int32 pad_id = 43 [default = -1]; bool has_pad_id() const; + private: + bool _internal_has_pad_id() const; + public: void clear_pad_id(); - static const int kPadIdFieldNumber = 43; - ::google::protobuf::int32 pad_id() const; - void set_pad_id(::google::protobuf::int32 value); + ::PROTOBUF_NAMESPACE_ID::int32 pad_id() const; + void set_pad_id(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_pad_id() const; + void _internal_set_pad_id(::PROTOBUF_NAMESPACE_ID::int32 value); + public: GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(TrainerSpec) // @@protoc_insertion_point(class_scope:sentencepiece.TrainerSpec) private: - void set_has_input_format(); - void clear_has_input_format(); - void set_has_model_prefix(); - void clear_has_model_prefix(); - void set_has_model_type(); - void clear_has_model_type(); - void set_has_vocab_size(); - void clear_has_vocab_size(); - void set_has_self_test_sample_size(); - void clear_has_self_test_sample_size(); - void set_has_character_coverage(); - void clear_has_character_coverage(); - void set_has_input_sentence_size(); - void clear_has_input_sentence_size(); - void set_has_shuffle_input_sentence(); - void clear_has_shuffle_input_sentence(); - void set_has_mining_sentence_size(); - void clear_has_mining_sentence_size(); - void set_has_training_sentence_size(); - void clear_has_training_sentence_size(); - void set_has_seed_sentencepiece_size(); - void clear_has_seed_sentencepiece_size(); - void set_has_shrinking_factor(); - void clear_has_shrinking_factor(); - void set_has_max_sentence_length(); - void clear_has_max_sentence_length(); - void set_has_num_threads(); - void clear_has_num_threads(); - void set_has_num_sub_iterations(); - void clear_has_num_sub_iterations(); - void set_has_max_sentencepiece_length(); - void clear_has_max_sentencepiece_length(); - void set_has_split_by_unicode_script(); - void clear_has_split_by_unicode_script(); - void set_has_split_by_number(); - void clear_has_split_by_number(); - void set_has_split_by_whitespace(); - void clear_has_split_by_whitespace(); - void set_has_treat_whitespace_as_suffix(); - void clear_has_treat_whitespace_as_suffix(); - void set_has_split_digits(); - void clear_has_split_digits(); - void set_has_required_chars(); - void clear_has_required_chars(); - void set_has_byte_fallback(); - void clear_has_byte_fallback(); - void set_has_vocabulary_output_piece_score(); - void clear_has_vocabulary_output_piece_score(); - void set_has_hard_vocab_limit(); - void clear_has_hard_vocab_limit(); - void set_has_use_all_vocab(); - void clear_has_use_all_vocab(); - void set_has_unk_id(); - void clear_has_unk_id(); - void set_has_bos_id(); - void clear_has_bos_id(); - void set_has_eos_id(); - void clear_has_eos_id(); - void set_has_pad_id(); - void clear_has_pad_id(); - void set_has_unk_piece(); - void clear_has_unk_piece(); - void set_has_bos_piece(); - void clear_has_bos_piece(); - void set_has_eos_piece(); - void clear_has_eos_piece(); - void set_has_pad_piece(); - void clear_has_pad_piece(); - void set_has_unk_surface(); - void clear_has_unk_surface(); - void set_has_train_extremely_large_corpus(); - void clear_has_train_extremely_large_corpus(); - - ::google::protobuf::internal::ExtensionSet _extensions_; - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<2> _has_bits_; - ::google::protobuf::RepeatedPtrField< ::std::string> input_; - ::google::protobuf::RepeatedPtrField< ::std::string> accept_language_; - ::google::protobuf::RepeatedPtrField< ::std::string> control_symbols_; - ::google::protobuf::RepeatedPtrField< ::std::string> user_defined_symbols_; - ::google::protobuf::internal::ArenaStringPtr model_prefix_; - ::google::protobuf::internal::ArenaStringPtr input_format_; - ::google::protobuf::internal::ArenaStringPtr required_chars_; - public: - static ::google::protobuf::internal::ExplicitlyConstructed< ::std::string> _i_give_permission_to_break_this_code_default_unk_surface_; - private: - ::google::protobuf::internal::ArenaStringPtr unk_surface_; - public: - static ::google::protobuf::internal::ExplicitlyConstructed< ::std::string> _i_give_permission_to_break_this_code_default_unk_piece_; - private: - ::google::protobuf::internal::ArenaStringPtr unk_piece_; - public: - static ::google::protobuf::internal::ExplicitlyConstructed< ::std::string> _i_give_permission_to_break_this_code_default_bos_piece_; - private: - ::google::protobuf::internal::ArenaStringPtr bos_piece_; - public: - static ::google::protobuf::internal::ExplicitlyConstructed< ::std::string> _i_give_permission_to_break_this_code_default_eos_piece_; - private: - ::google::protobuf::internal::ArenaStringPtr eos_piece_; - public: - static ::google::protobuf::internal::ExplicitlyConstructed< ::std::string> _i_give_permission_to_break_this_code_default_pad_piece_; - private: - ::google::protobuf::internal::ArenaStringPtr pad_piece_; - ::google::protobuf::int32 self_test_sample_size_; - ::google::protobuf::int32 input_sentence_size_; - ::google::protobuf::int32 mining_sentence_size_; - ::google::protobuf::int32 training_sentence_size_; + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::ExtensionSet _extensions_; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<2> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField input_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField accept_language_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField control_symbols_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField user_defined_symbols_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr model_prefix_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr input_format_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr required_chars_; + public: + static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _i_give_permission_to_break_this_code_default_unk_surface_; + private: + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr unk_surface_; + public: + static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _i_give_permission_to_break_this_code_default_unk_piece_; + private: + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr unk_piece_; + public: + static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _i_give_permission_to_break_this_code_default_bos_piece_; + private: + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr bos_piece_; + public: + static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _i_give_permission_to_break_this_code_default_eos_piece_; + private: + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr eos_piece_; + public: + static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _i_give_permission_to_break_this_code_default_pad_piece_; + private: + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr pad_piece_; + ::PROTOBUF_NAMESPACE_ID::int32 self_test_sample_size_; + ::PROTOBUF_NAMESPACE_ID::int32 input_sentence_size_; + ::PROTOBUF_NAMESPACE_ID::int32 mining_sentence_size_; + ::PROTOBUF_NAMESPACE_ID::int32 training_sentence_size_; bool treat_whitespace_as_suffix_; bool split_digits_; bool byte_fallback_; bool use_all_vocab_; - ::google::protobuf::int32 unk_id_; + ::PROTOBUF_NAMESPACE_ID::int32 unk_id_; bool train_extremely_large_corpus_; int model_type_; - ::google::protobuf::int32 vocab_size_; + ::PROTOBUF_NAMESPACE_ID::int32 vocab_size_; float character_coverage_; - ::google::protobuf::int32 seed_sentencepiece_size_; + ::PROTOBUF_NAMESPACE_ID::int32 seed_sentencepiece_size_; float shrinking_factor_; - ::google::protobuf::int32 num_threads_; - ::google::protobuf::int32 num_sub_iterations_; - ::google::protobuf::int32 max_sentence_length_; - ::google::protobuf::int32 max_sentencepiece_length_; + ::PROTOBUF_NAMESPACE_ID::int32 num_threads_; + ::PROTOBUF_NAMESPACE_ID::int32 num_sub_iterations_; + ::PROTOBUF_NAMESPACE_ID::int32 max_sentence_length_; + ::PROTOBUF_NAMESPACE_ID::int32 max_sentencepiece_length_; bool shuffle_input_sentence_; bool split_by_unicode_script_; bool split_by_number_; bool split_by_whitespace_; bool vocabulary_output_piece_score_; bool hard_vocab_limit_; - ::google::protobuf::int32 bos_id_; - ::google::protobuf::int32 eos_id_; - ::google::protobuf::int32 pad_id_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - friend struct ::protobuf_sentencepiece_5fmodel_2eproto::TableStruct; + ::PROTOBUF_NAMESPACE_ID::int32 bos_id_; + ::PROTOBUF_NAMESPACE_ID::int32 eos_id_; + ::PROTOBUF_NAMESPACE_ID::int32 pad_id_; + friend struct ::TableStruct_sentencepiece_5fmodel_2eproto; }; // ------------------------------------------------------------------- -class NormalizerSpec : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.NormalizerSpec) */ { +class NormalizerSpec PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.NormalizerSpec) */ { public: - NormalizerSpec(); + inline NormalizerSpec() : NormalizerSpec(nullptr) {}; virtual ~NormalizerSpec(); NormalizerSpec(const NormalizerSpec& from); - - inline NormalizerSpec& operator=(const NormalizerSpec& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 NormalizerSpec(NormalizerSpec&& from) noexcept : NormalizerSpec() { *this = ::std::move(from); } + inline NormalizerSpec& operator=(const NormalizerSpec& from) { + CopyFrom(from); + return *this; + } inline NormalizerSpec& operator=(NormalizerSpec&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const NormalizerSpec& default_instance(); @@ -800,183 +1109,254 @@ class NormalizerSpec : public ::google::protobuf::MessageLite /* @@protoc_insert static constexpr int kIndexInFileMessages = 1; - void Swap(NormalizerSpec* other); friend void swap(NormalizerSpec& a, NormalizerSpec& b) { a.Swap(&b); } + inline void Swap(NormalizerSpec* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(NormalizerSpec* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline NormalizerSpec* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - NormalizerSpec* New(::google::protobuf::Arena* arena) const final { + NormalizerSpec* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const NormalizerSpec& from); void MergeFrom(const NormalizerSpec& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(NormalizerSpec* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.NormalizerSpec"; } + protected: + explicit NormalizerSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- + enum : int { + kNameFieldNumber = 1, + kPrecompiledCharsmapFieldNumber = 2, + kNormalizationRuleTsvFieldNumber = 6, + kAddDummyPrefixFieldNumber = 3, + kRemoveExtraWhitespacesFieldNumber = 4, + kEscapeWhitespacesFieldNumber = 5, + }; // optional string name = 1; bool has_name() const; + private: + bool _internal_has_name() const; + public: void clear_name(); - static const int kNameFieldNumber = 1; - const ::std::string& name() const; - void set_name(const ::std::string& value); - #if LANG_CXX11 - void set_name(::std::string&& value); - #endif + const std::string& name() const; + void set_name(const std::string& value); + void set_name(std::string&& value); void set_name(const char* value); void set_name(const char* value, size_t size); - ::std::string* mutable_name(); - ::std::string* release_name(); - void set_allocated_name(::std::string* name); + std::string* mutable_name(); + std::string* release_name(); + void set_allocated_name(std::string* name); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_name(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_name( + std::string* name); + private: + const std::string& _internal_name() const; + void _internal_set_name(const std::string& value); + std::string* _internal_mutable_name(); + public: // optional bytes precompiled_charsmap = 2; bool has_precompiled_charsmap() const; + private: + bool _internal_has_precompiled_charsmap() const; + public: void clear_precompiled_charsmap(); - static const int kPrecompiledCharsmapFieldNumber = 2; - const ::std::string& precompiled_charsmap() const; - void set_precompiled_charsmap(const ::std::string& value); - #if LANG_CXX11 - void set_precompiled_charsmap(::std::string&& value); - #endif + const std::string& precompiled_charsmap() const; + void set_precompiled_charsmap(const std::string& value); + void set_precompiled_charsmap(std::string&& value); void set_precompiled_charsmap(const char* value); void set_precompiled_charsmap(const void* value, size_t size); - ::std::string* mutable_precompiled_charsmap(); - ::std::string* release_precompiled_charsmap(); - void set_allocated_precompiled_charsmap(::std::string* precompiled_charsmap); + std::string* mutable_precompiled_charsmap(); + std::string* release_precompiled_charsmap(); + void set_allocated_precompiled_charsmap(std::string* precompiled_charsmap); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_precompiled_charsmap(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_precompiled_charsmap( + std::string* precompiled_charsmap); + private: + const std::string& _internal_precompiled_charsmap() const; + void _internal_set_precompiled_charsmap(const std::string& value); + std::string* _internal_mutable_precompiled_charsmap(); + public: // optional string normalization_rule_tsv = 6; bool has_normalization_rule_tsv() const; + private: + bool _internal_has_normalization_rule_tsv() const; + public: void clear_normalization_rule_tsv(); - static const int kNormalizationRuleTsvFieldNumber = 6; - const ::std::string& normalization_rule_tsv() const; - void set_normalization_rule_tsv(const ::std::string& value); - #if LANG_CXX11 - void set_normalization_rule_tsv(::std::string&& value); - #endif + const std::string& normalization_rule_tsv() const; + void set_normalization_rule_tsv(const std::string& value); + void set_normalization_rule_tsv(std::string&& value); void set_normalization_rule_tsv(const char* value); void set_normalization_rule_tsv(const char* value, size_t size); - ::std::string* mutable_normalization_rule_tsv(); - ::std::string* release_normalization_rule_tsv(); - void set_allocated_normalization_rule_tsv(::std::string* normalization_rule_tsv); + std::string* mutable_normalization_rule_tsv(); + std::string* release_normalization_rule_tsv(); + void set_allocated_normalization_rule_tsv(std::string* normalization_rule_tsv); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_normalization_rule_tsv(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_normalization_rule_tsv( + std::string* normalization_rule_tsv); + private: + const std::string& _internal_normalization_rule_tsv() const; + void _internal_set_normalization_rule_tsv(const std::string& value); + std::string* _internal_mutable_normalization_rule_tsv(); + public: // optional bool add_dummy_prefix = 3 [default = true]; bool has_add_dummy_prefix() const; + private: + bool _internal_has_add_dummy_prefix() const; + public: void clear_add_dummy_prefix(); - static const int kAddDummyPrefixFieldNumber = 3; bool add_dummy_prefix() const; void set_add_dummy_prefix(bool value); + private: + bool _internal_add_dummy_prefix() const; + void _internal_set_add_dummy_prefix(bool value); + public: // optional bool remove_extra_whitespaces = 4 [default = true]; bool has_remove_extra_whitespaces() const; + private: + bool _internal_has_remove_extra_whitespaces() const; + public: void clear_remove_extra_whitespaces(); - static const int kRemoveExtraWhitespacesFieldNumber = 4; bool remove_extra_whitespaces() const; void set_remove_extra_whitespaces(bool value); + private: + bool _internal_remove_extra_whitespaces() const; + void _internal_set_remove_extra_whitespaces(bool value); + public: // optional bool escape_whitespaces = 5 [default = true]; bool has_escape_whitespaces() const; + private: + bool _internal_has_escape_whitespaces() const; + public: void clear_escape_whitespaces(); - static const int kEscapeWhitespacesFieldNumber = 5; bool escape_whitespaces() const; void set_escape_whitespaces(bool value); + private: + bool _internal_escape_whitespaces() const; + void _internal_set_escape_whitespaces(bool value); + public: GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(NormalizerSpec) // @@protoc_insertion_point(class_scope:sentencepiece.NormalizerSpec) private: - void set_has_name(); - void clear_has_name(); - void set_has_precompiled_charsmap(); - void clear_has_precompiled_charsmap(); - void set_has_add_dummy_prefix(); - void clear_has_add_dummy_prefix(); - void set_has_remove_extra_whitespaces(); - void clear_has_remove_extra_whitespaces(); - void set_has_escape_whitespaces(); - void clear_has_escape_whitespaces(); - void set_has_normalization_rule_tsv(); - void clear_has_normalization_rule_tsv(); - - ::google::protobuf::internal::ExtensionSet _extensions_; - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::internal::ArenaStringPtr name_; - ::google::protobuf::internal::ArenaStringPtr precompiled_charsmap_; - ::google::protobuf::internal::ArenaStringPtr normalization_rule_tsv_; + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::ExtensionSet _extensions_; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr precompiled_charsmap_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr normalization_rule_tsv_; bool add_dummy_prefix_; bool remove_extra_whitespaces_; bool escape_whitespaces_; - friend struct ::protobuf_sentencepiece_5fmodel_2eproto::TableStruct; + friend struct ::TableStruct_sentencepiece_5fmodel_2eproto; }; // ------------------------------------------------------------------- -class SelfTestData_Sample : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SelfTestData.Sample) */ { +class SelfTestData_Sample PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SelfTestData.Sample) */ { public: - SelfTestData_Sample(); + inline SelfTestData_Sample() : SelfTestData_Sample(nullptr) {}; virtual ~SelfTestData_Sample(); SelfTestData_Sample(const SelfTestData_Sample& from); - - inline SelfTestData_Sample& operator=(const SelfTestData_Sample& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 SelfTestData_Sample(SelfTestData_Sample&& from) noexcept : SelfTestData_Sample() { *this = ::std::move(from); } + inline SelfTestData_Sample& operator=(const SelfTestData_Sample& from) { + CopyFrom(from); + return *this; + } inline SelfTestData_Sample& operator=(SelfTestData_Sample&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const SelfTestData_Sample& default_instance(); @@ -989,132 +1369,175 @@ class SelfTestData_Sample : public ::google::protobuf::MessageLite /* @@protoc_i static constexpr int kIndexInFileMessages = 2; - void Swap(SelfTestData_Sample* other); friend void swap(SelfTestData_Sample& a, SelfTestData_Sample& b) { a.Swap(&b); } + inline void Swap(SelfTestData_Sample* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(SelfTestData_Sample* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline SelfTestData_Sample* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - SelfTestData_Sample* New(::google::protobuf::Arena* arena) const final { + SelfTestData_Sample* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const SelfTestData_Sample& from); void MergeFrom(const SelfTestData_Sample& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(SelfTestData_Sample* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.SelfTestData.Sample"; } + protected: + explicit SelfTestData_Sample(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- + enum : int { + kInputFieldNumber = 1, + kExpectedFieldNumber = 2, + }; // optional string input = 1; bool has_input() const; + private: + bool _internal_has_input() const; + public: void clear_input(); - static const int kInputFieldNumber = 1; - const ::std::string& input() const; - void set_input(const ::std::string& value); - #if LANG_CXX11 - void set_input(::std::string&& value); - #endif + const std::string& input() const; + void set_input(const std::string& value); + void set_input(std::string&& value); void set_input(const char* value); void set_input(const char* value, size_t size); - ::std::string* mutable_input(); - ::std::string* release_input(); - void set_allocated_input(::std::string* input); + std::string* mutable_input(); + std::string* release_input(); + void set_allocated_input(std::string* input); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_input(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_input( + std::string* input); + private: + const std::string& _internal_input() const; + void _internal_set_input(const std::string& value); + std::string* _internal_mutable_input(); + public: // optional string expected = 2; bool has_expected() const; + private: + bool _internal_has_expected() const; + public: void clear_expected(); - static const int kExpectedFieldNumber = 2; - const ::std::string& expected() const; - void set_expected(const ::std::string& value); - #if LANG_CXX11 - void set_expected(::std::string&& value); - #endif + const std::string& expected() const; + void set_expected(const std::string& value); + void set_expected(std::string&& value); void set_expected(const char* value); void set_expected(const char* value, size_t size); - ::std::string* mutable_expected(); - ::std::string* release_expected(); - void set_allocated_expected(::std::string* expected); + std::string* mutable_expected(); + std::string* release_expected(); + void set_allocated_expected(std::string* expected); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_expected(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_expected( + std::string* expected); + private: + const std::string& _internal_expected() const; + void _internal_set_expected(const std::string& value); + std::string* _internal_mutable_expected(); + public: // @@protoc_insertion_point(class_scope:sentencepiece.SelfTestData.Sample) private: - void set_has_input(); - void clear_has_input(); - void set_has_expected(); - void clear_has_expected(); - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::internal::ArenaStringPtr input_; - ::google::protobuf::internal::ArenaStringPtr expected_; - friend struct ::protobuf_sentencepiece_5fmodel_2eproto::TableStruct; + class _Internal; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr input_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr expected_; + friend struct ::TableStruct_sentencepiece_5fmodel_2eproto; }; // ------------------------------------------------------------------- -class SelfTestData : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SelfTestData) */ { +class SelfTestData PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.SelfTestData) */ { public: - SelfTestData(); + inline SelfTestData() : SelfTestData(nullptr) {}; virtual ~SelfTestData(); SelfTestData(const SelfTestData& from); - - inline SelfTestData& operator=(const SelfTestData& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 SelfTestData(SelfTestData&& from) noexcept : SelfTestData() { *this = ::std::move(from); } + inline SelfTestData& operator=(const SelfTestData& from) { + CopyFrom(from); + return *this; + } inline SelfTestData& operator=(SelfTestData&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const SelfTestData& default_instance(); @@ -1127,50 +1550,63 @@ class SelfTestData : public ::google::protobuf::MessageLite /* @@protoc_insertio static constexpr int kIndexInFileMessages = 3; - void Swap(SelfTestData* other); friend void swap(SelfTestData& a, SelfTestData& b) { a.Swap(&b); } + inline void Swap(SelfTestData* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(SelfTestData* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline SelfTestData* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - SelfTestData* New(::google::protobuf::Arena* arena) const final { + SelfTestData* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const SelfTestData& from); void MergeFrom(const SelfTestData& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(SelfTestData* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.SelfTestData"; } + protected: + explicit SelfTestData(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- @@ -1178,63 +1614,73 @@ class SelfTestData : public ::google::protobuf::MessageLite /* @@protoc_insertio // accessors ------------------------------------------------------- + enum : int { + kSamplesFieldNumber = 1, + }; // repeated .sentencepiece.SelfTestData.Sample samples = 1; int samples_size() const; + private: + int _internal_samples_size() const; + public: void clear_samples(); - static const int kSamplesFieldNumber = 1; ::sentencepiece::SelfTestData_Sample* mutable_samples(int index); - ::google::protobuf::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >* + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >* mutable_samples(); + private: + const ::sentencepiece::SelfTestData_Sample& _internal_samples(int index) const; + ::sentencepiece::SelfTestData_Sample* _internal_add_samples(); + public: const ::sentencepiece::SelfTestData_Sample& samples(int index) const; ::sentencepiece::SelfTestData_Sample* add_samples(); - const ::google::protobuf::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >& + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >& samples() const; GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(SelfTestData) // @@protoc_insertion_point(class_scope:sentencepiece.SelfTestData) private: + class _Internal; - ::google::protobuf::internal::ExtensionSet _extensions_; + ::PROTOBUF_NAMESPACE_ID::internal::ExtensionSet _extensions_; - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample > samples_; - friend struct ::protobuf_sentencepiece_5fmodel_2eproto::TableStruct; + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample > samples_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_sentencepiece_5fmodel_2eproto; }; // ------------------------------------------------------------------- -class ModelProto_SentencePiece : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.ModelProto.SentencePiece) */ { +class ModelProto_SentencePiece PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.ModelProto.SentencePiece) */ { public: - ModelProto_SentencePiece(); + inline ModelProto_SentencePiece() : ModelProto_SentencePiece(nullptr) {}; virtual ~ModelProto_SentencePiece(); ModelProto_SentencePiece(const ModelProto_SentencePiece& from); - - inline ModelProto_SentencePiece& operator=(const ModelProto_SentencePiece& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 ModelProto_SentencePiece(ModelProto_SentencePiece&& from) noexcept : ModelProto_SentencePiece() { *this = ::std::move(from); } + inline ModelProto_SentencePiece& operator=(const ModelProto_SentencePiece& from) { + CopyFrom(from); + return *this; + } inline ModelProto_SentencePiece& operator=(ModelProto_SentencePiece&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const ModelProto_SentencePiece& default_instance(); @@ -1247,160 +1693,211 @@ class ModelProto_SentencePiece : public ::google::protobuf::MessageLite /* @@pro static constexpr int kIndexInFileMessages = 4; - void Swap(ModelProto_SentencePiece* other); friend void swap(ModelProto_SentencePiece& a, ModelProto_SentencePiece& b) { a.Swap(&b); } + inline void Swap(ModelProto_SentencePiece* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(ModelProto_SentencePiece* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline ModelProto_SentencePiece* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - ModelProto_SentencePiece* New(::google::protobuf::Arena* arena) const final { + ModelProto_SentencePiece* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const ModelProto_SentencePiece& from); void MergeFrom(const ModelProto_SentencePiece& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(ModelProto_SentencePiece* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.ModelProto.SentencePiece"; } + protected: + explicit ModelProto_SentencePiece(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- typedef ModelProto_SentencePiece_Type Type; - static const Type NORMAL = + static constexpr Type NORMAL = ModelProto_SentencePiece_Type_NORMAL; - static const Type UNKNOWN = + static constexpr Type UNKNOWN = ModelProto_SentencePiece_Type_UNKNOWN; - static const Type CONTROL = + static constexpr Type CONTROL = ModelProto_SentencePiece_Type_CONTROL; - static const Type USER_DEFINED = + static constexpr Type USER_DEFINED = ModelProto_SentencePiece_Type_USER_DEFINED; - static const Type BYTE = + static constexpr Type BYTE = ModelProto_SentencePiece_Type_BYTE; - static const Type UNUSED = + static constexpr Type UNUSED = ModelProto_SentencePiece_Type_UNUSED; static inline bool Type_IsValid(int value) { return ModelProto_SentencePiece_Type_IsValid(value); } - static const Type Type_MIN = + static constexpr Type Type_MIN = ModelProto_SentencePiece_Type_Type_MIN; - static const Type Type_MAX = + static constexpr Type Type_MAX = ModelProto_SentencePiece_Type_Type_MAX; - static const int Type_ARRAYSIZE = + static constexpr int Type_ARRAYSIZE = ModelProto_SentencePiece_Type_Type_ARRAYSIZE; + template + static inline const std::string& Type_Name(T enum_t_value) { + static_assert(::std::is_same::value || + ::std::is_integral::value, + "Incorrect type passed to function Type_Name."); + return ModelProto_SentencePiece_Type_Name(enum_t_value); + } + static inline bool Type_Parse(const std::string& name, + Type* value) { + return ModelProto_SentencePiece_Type_Parse(name, value); + } // accessors ------------------------------------------------------- + enum : int { + kPieceFieldNumber = 1, + kScoreFieldNumber = 2, + kTypeFieldNumber = 3, + }; // optional string piece = 1; bool has_piece() const; + private: + bool _internal_has_piece() const; + public: void clear_piece(); - static const int kPieceFieldNumber = 1; - const ::std::string& piece() const; - void set_piece(const ::std::string& value); - #if LANG_CXX11 - void set_piece(::std::string&& value); - #endif + const std::string& piece() const; + void set_piece(const std::string& value); + void set_piece(std::string&& value); void set_piece(const char* value); void set_piece(const char* value, size_t size); - ::std::string* mutable_piece(); - ::std::string* release_piece(); - void set_allocated_piece(::std::string* piece); + std::string* mutable_piece(); + std::string* release_piece(); + void set_allocated_piece(std::string* piece); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + std::string* unsafe_arena_release_piece(); + GOOGLE_PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for" + " string fields are deprecated and will be removed in a" + " future release.") + void unsafe_arena_set_allocated_piece( + std::string* piece); + private: + const std::string& _internal_piece() const; + void _internal_set_piece(const std::string& value); + std::string* _internal_mutable_piece(); + public: // optional float score = 2; bool has_score() const; + private: + bool _internal_has_score() const; + public: void clear_score(); - static const int kScoreFieldNumber = 2; float score() const; void set_score(float value); + private: + float _internal_score() const; + void _internal_set_score(float value); + public: // optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default = NORMAL]; bool has_type() const; + private: + bool _internal_has_type() const; + public: void clear_type(); - static const int kTypeFieldNumber = 3; ::sentencepiece::ModelProto_SentencePiece_Type type() const; void set_type(::sentencepiece::ModelProto_SentencePiece_Type value); + private: + ::sentencepiece::ModelProto_SentencePiece_Type _internal_type() const; + void _internal_set_type(::sentencepiece::ModelProto_SentencePiece_Type value); + public: GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(ModelProto_SentencePiece) // @@protoc_insertion_point(class_scope:sentencepiece.ModelProto.SentencePiece) private: - void set_has_piece(); - void clear_has_piece(); - void set_has_score(); - void clear_has_score(); - void set_has_type(); - void clear_has_type(); - - ::google::protobuf::internal::ExtensionSet _extensions_; - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::internal::ArenaStringPtr piece_; + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::ExtensionSet _extensions_; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr piece_; float score_; int type_; - friend struct ::protobuf_sentencepiece_5fmodel_2eproto::TableStruct; + friend struct ::TableStruct_sentencepiece_5fmodel_2eproto; }; // ------------------------------------------------------------------- -class ModelProto : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.ModelProto) */ { +class ModelProto PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:sentencepiece.ModelProto) */ { public: - ModelProto(); + inline ModelProto() : ModelProto(nullptr) {}; virtual ~ModelProto(); ModelProto(const ModelProto& from); - - inline ModelProto& operator=(const ModelProto& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 ModelProto(ModelProto&& from) noexcept : ModelProto() { *this = ::std::move(from); } + inline ModelProto& operator=(const ModelProto& from) { + CopyFrom(from); + return *this; + } inline ModelProto& operator=(ModelProto&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (GetArena() == from.GetArena()) { if (this != &from) InternalSwap(&from); } else { CopyFrom(from); } return *this; } - #endif - inline const ::std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(); + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); } - inline ::std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); } static const ModelProto& default_instance(); @@ -1413,50 +1910,63 @@ class ModelProto : public ::google::protobuf::MessageLite /* @@protoc_insertion_ static constexpr int kIndexInFileMessages = 5; - void Swap(ModelProto* other); friend void swap(ModelProto& a, ModelProto& b) { a.Swap(&b); } + inline void Swap(ModelProto* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(ModelProto* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } // implements Message ---------------------------------------------- inline ModelProto* New() const final { - return CreateMaybeMessage(NULL); + return CreateMaybeMessage(nullptr); } - ModelProto* New(::google::protobuf::Arena* arena) const final { + ModelProto* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { return CreateMaybeMessage(arena); } - void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; void CopyFrom(const ModelProto& from); void MergeFrom(const ModelProto& from); - void Clear() final; + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) final; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; void DiscardUnknownFields(); int GetCachedSize() const final { return _cached_size_.Get(); } private: - void SharedCtor(); - void SharedDtor(); + inline void SharedCtor(); + inline void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(ModelProto* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "sentencepiece.ModelProto"; } + protected: + explicit ModelProto(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); public: - ::std::string GetTypeName() const final; + std::string GetTypeName() const final; // nested types ---------------------------------------------------- @@ -1464,89 +1974,121 @@ class ModelProto : public ::google::protobuf::MessageLite /* @@protoc_insertion_ // accessors ------------------------------------------------------- + enum : int { + kPiecesFieldNumber = 1, + kTrainerSpecFieldNumber = 2, + kNormalizerSpecFieldNumber = 3, + kSelfTestDataFieldNumber = 4, + kDenormalizerSpecFieldNumber = 5, + }; // repeated .sentencepiece.ModelProto.SentencePiece pieces = 1; int pieces_size() const; + private: + int _internal_pieces_size() const; + public: void clear_pieces(); - static const int kPiecesFieldNumber = 1; ::sentencepiece::ModelProto_SentencePiece* mutable_pieces(int index); - ::google::protobuf::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >* + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >* mutable_pieces(); + private: + const ::sentencepiece::ModelProto_SentencePiece& _internal_pieces(int index) const; + ::sentencepiece::ModelProto_SentencePiece* _internal_add_pieces(); + public: const ::sentencepiece::ModelProto_SentencePiece& pieces(int index) const; ::sentencepiece::ModelProto_SentencePiece* add_pieces(); - const ::google::protobuf::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >& + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >& pieces() const; // optional .sentencepiece.TrainerSpec trainer_spec = 2; bool has_trainer_spec() const; - void clear_trainer_spec(); - static const int kTrainerSpecFieldNumber = 2; private: - const ::sentencepiece::TrainerSpec& _internal_trainer_spec() const; + bool _internal_has_trainer_spec() const; public: + void clear_trainer_spec(); const ::sentencepiece::TrainerSpec& trainer_spec() const; ::sentencepiece::TrainerSpec* release_trainer_spec(); ::sentencepiece::TrainerSpec* mutable_trainer_spec(); void set_allocated_trainer_spec(::sentencepiece::TrainerSpec* trainer_spec); + private: + const ::sentencepiece::TrainerSpec& _internal_trainer_spec() const; + ::sentencepiece::TrainerSpec* _internal_mutable_trainer_spec(); + public: + void unsafe_arena_set_allocated_trainer_spec( + ::sentencepiece::TrainerSpec* trainer_spec); + ::sentencepiece::TrainerSpec* unsafe_arena_release_trainer_spec(); // optional .sentencepiece.NormalizerSpec normalizer_spec = 3; bool has_normalizer_spec() const; - void clear_normalizer_spec(); - static const int kNormalizerSpecFieldNumber = 3; private: - const ::sentencepiece::NormalizerSpec& _internal_normalizer_spec() const; + bool _internal_has_normalizer_spec() const; public: + void clear_normalizer_spec(); const ::sentencepiece::NormalizerSpec& normalizer_spec() const; ::sentencepiece::NormalizerSpec* release_normalizer_spec(); ::sentencepiece::NormalizerSpec* mutable_normalizer_spec(); void set_allocated_normalizer_spec(::sentencepiece::NormalizerSpec* normalizer_spec); + private: + const ::sentencepiece::NormalizerSpec& _internal_normalizer_spec() const; + ::sentencepiece::NormalizerSpec* _internal_mutable_normalizer_spec(); + public: + void unsafe_arena_set_allocated_normalizer_spec( + ::sentencepiece::NormalizerSpec* normalizer_spec); + ::sentencepiece::NormalizerSpec* unsafe_arena_release_normalizer_spec(); // optional .sentencepiece.SelfTestData self_test_data = 4; bool has_self_test_data() const; - void clear_self_test_data(); - static const int kSelfTestDataFieldNumber = 4; private: - const ::sentencepiece::SelfTestData& _internal_self_test_data() const; + bool _internal_has_self_test_data() const; public: + void clear_self_test_data(); const ::sentencepiece::SelfTestData& self_test_data() const; ::sentencepiece::SelfTestData* release_self_test_data(); ::sentencepiece::SelfTestData* mutable_self_test_data(); void set_allocated_self_test_data(::sentencepiece::SelfTestData* self_test_data); + private: + const ::sentencepiece::SelfTestData& _internal_self_test_data() const; + ::sentencepiece::SelfTestData* _internal_mutable_self_test_data(); + public: + void unsafe_arena_set_allocated_self_test_data( + ::sentencepiece::SelfTestData* self_test_data); + ::sentencepiece::SelfTestData* unsafe_arena_release_self_test_data(); // optional .sentencepiece.NormalizerSpec denormalizer_spec = 5; bool has_denormalizer_spec() const; - void clear_denormalizer_spec(); - static const int kDenormalizerSpecFieldNumber = 5; private: - const ::sentencepiece::NormalizerSpec& _internal_denormalizer_spec() const; + bool _internal_has_denormalizer_spec() const; public: + void clear_denormalizer_spec(); const ::sentencepiece::NormalizerSpec& denormalizer_spec() const; ::sentencepiece::NormalizerSpec* release_denormalizer_spec(); ::sentencepiece::NormalizerSpec* mutable_denormalizer_spec(); void set_allocated_denormalizer_spec(::sentencepiece::NormalizerSpec* denormalizer_spec); + private: + const ::sentencepiece::NormalizerSpec& _internal_denormalizer_spec() const; + ::sentencepiece::NormalizerSpec* _internal_mutable_denormalizer_spec(); + public: + void unsafe_arena_set_allocated_denormalizer_spec( + ::sentencepiece::NormalizerSpec* denormalizer_spec); + ::sentencepiece::NormalizerSpec* unsafe_arena_release_denormalizer_spec(); GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(ModelProto) // @@protoc_insertion_point(class_scope:sentencepiece.ModelProto) private: - void set_has_trainer_spec(); - void clear_has_trainer_spec(); - void set_has_normalizer_spec(); - void clear_has_normalizer_spec(); - void set_has_self_test_data(); - void clear_has_self_test_data(); - void set_has_denormalizer_spec(); - void clear_has_denormalizer_spec(); - - ::google::protobuf::internal::ExtensionSet _extensions_; - - ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - ::google::protobuf::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece > pieces_; + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::ExtensionSet _extensions_; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece > pieces_; ::sentencepiece::TrainerSpec* trainer_spec_; ::sentencepiece::NormalizerSpec* normalizer_spec_; ::sentencepiece::SelfTestData* self_test_data_; ::sentencepiece::NormalizerSpec* denormalizer_spec_; - friend struct ::protobuf_sentencepiece_5fmodel_2eproto::TableStruct; + friend struct ::TableStruct_sentencepiece_5fmodel_2eproto; }; // =================================================================== @@ -1560,32 +2102,40 @@ class ModelProto : public ::google::protobuf::MessageLite /* @@protoc_insertion_ // TrainerSpec // repeated string input = 1; -inline int TrainerSpec::input_size() const { +inline int TrainerSpec::_internal_input_size() const { return input_.size(); } +inline int TrainerSpec::input_size() const { + return _internal_input_size(); +} inline void TrainerSpec::clear_input() { input_.Clear(); } -inline const ::std::string& TrainerSpec::input(int index) const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.input) +inline std::string* TrainerSpec::add_input() { + // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.input) + return _internal_add_input(); +} +inline const std::string& TrainerSpec::_internal_input(int index) const { return input_.Get(index); } -inline ::std::string* TrainerSpec::mutable_input(int index) { +inline const std::string& TrainerSpec::input(int index) const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.input) + return _internal_input(index); +} +inline std::string* TrainerSpec::mutable_input(int index) { // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.input) return input_.Mutable(index); } -inline void TrainerSpec::set_input(int index, const ::std::string& value) { +inline void TrainerSpec::set_input(int index, const std::string& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.input) input_.Mutable(index)->assign(value); } -#if LANG_CXX11 -inline void TrainerSpec::set_input(int index, ::std::string&& value) { +inline void TrainerSpec::set_input(int index, std::string&& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.input) input_.Mutable(index)->assign(std::move(value)); } -#endif inline void TrainerSpec::set_input(int index, const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); input_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.input) } @@ -1594,22 +2144,19 @@ inline void TrainerSpec::set_input(int index, const char* value, size_t size) { reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.input) } -inline ::std::string* TrainerSpec::add_input() { - // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.input) +inline std::string* TrainerSpec::_internal_add_input() { return input_.Add(); } -inline void TrainerSpec::add_input(const ::std::string& value) { +inline void TrainerSpec::add_input(const std::string& value) { input_.Add()->assign(value); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.input) } -#if LANG_CXX11 -inline void TrainerSpec::add_input(::std::string&& value) { +inline void TrainerSpec::add_input(std::string&& value) { input_.Add(std::move(value)); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.input) } -#endif inline void TrainerSpec::add_input(const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); input_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:sentencepiece.TrainerSpec.input) } @@ -1617,225 +2164,295 @@ inline void TrainerSpec::add_input(const char* value, size_t size) { input_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:sentencepiece.TrainerSpec.input) } -inline const ::google::protobuf::RepeatedPtrField< ::std::string>& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& TrainerSpec::input() const { // @@protoc_insertion_point(field_list:sentencepiece.TrainerSpec.input) return input_; } -inline ::google::protobuf::RepeatedPtrField< ::std::string>* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* TrainerSpec::mutable_input() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.TrainerSpec.input) return &input_; } // optional string input_format = 7; -inline bool TrainerSpec::has_input_format() const { - return (_has_bits_[0] & 0x00000002u) != 0; -} -inline void TrainerSpec::set_has_input_format() { - _has_bits_[0] |= 0x00000002u; +inline bool TrainerSpec::_internal_has_input_format() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + return value; } -inline void TrainerSpec::clear_has_input_format() { - _has_bits_[0] &= ~0x00000002u; +inline bool TrainerSpec::has_input_format() const { + return _internal_has_input_format(); } inline void TrainerSpec::clear_input_format() { - input_format_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_input_format(); + input_format_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000002u; } -inline const ::std::string& TrainerSpec::input_format() const { +inline const std::string& TrainerSpec::input_format() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.input_format) - return input_format_.GetNoArena(); + return _internal_input_format(); } -inline void TrainerSpec::set_input_format(const ::std::string& value) { - set_has_input_format(); - input_format_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void TrainerSpec::set_input_format(const std::string& value) { + _internal_set_input_format(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.input_format) } -#if LANG_CXX11 -inline void TrainerSpec::set_input_format(::std::string&& value) { - set_has_input_format(); - input_format_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_input_format() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.input_format) + return _internal_mutable_input_format(); +} +inline const std::string& TrainerSpec::_internal_input_format() const { + return input_format_.Get(); +} +inline void TrainerSpec::_internal_set_input_format(const std::string& value) { + _has_bits_[0] |= 0x00000002u; + input_format_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void TrainerSpec::set_input_format(std::string&& value) { + _has_bits_[0] |= 0x00000002u; + input_format_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.input_format) } -#endif inline void TrainerSpec::set_input_format(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_input_format(); - input_format_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000002u; + input_format_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.input_format) } -inline void TrainerSpec::set_input_format(const char* value, size_t size) { - set_has_input_format(); - input_format_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_input_format(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000002u; + input_format_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.input_format) } -inline ::std::string* TrainerSpec::mutable_input_format() { - set_has_input_format(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.input_format) - return input_format_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* TrainerSpec::_internal_mutable_input_format() { + _has_bits_[0] |= 0x00000002u; + return input_format_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* TrainerSpec::release_input_format() { +inline std::string* TrainerSpec::release_input_format() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.input_format) - if (!has_input_format()) { - return NULL; + if (!_internal_has_input_format()) { + return nullptr; } - clear_has_input_format(); - return input_format_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000002u; + return input_format_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void TrainerSpec::set_allocated_input_format(::std::string* input_format) { - if (input_format != NULL) { - set_has_input_format(); +inline void TrainerSpec::set_allocated_input_format(std::string* input_format) { + if (input_format != nullptr) { + _has_bits_[0] |= 0x00000002u; } else { - clear_has_input_format(); + _has_bits_[0] &= ~0x00000002u; } - input_format_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), input_format); + input_format_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), input_format, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.input_format) } +inline std::string* TrainerSpec::unsafe_arena_release_input_format() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.input_format) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000002u; + return input_format_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_input_format( + std::string* input_format) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (input_format != nullptr) { + _has_bits_[0] |= 0x00000002u; + } else { + _has_bits_[0] &= ~0x00000002u; + } + input_format_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + input_format, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.input_format) +} // optional string model_prefix = 2; -inline bool TrainerSpec::has_model_prefix() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void TrainerSpec::set_has_model_prefix() { - _has_bits_[0] |= 0x00000001u; +inline bool TrainerSpec::_internal_has_model_prefix() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + return value; } -inline void TrainerSpec::clear_has_model_prefix() { - _has_bits_[0] &= ~0x00000001u; +inline bool TrainerSpec::has_model_prefix() const { + return _internal_has_model_prefix(); } inline void TrainerSpec::clear_model_prefix() { - model_prefix_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_model_prefix(); + model_prefix_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000001u; } -inline const ::std::string& TrainerSpec::model_prefix() const { +inline const std::string& TrainerSpec::model_prefix() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.model_prefix) - return model_prefix_.GetNoArena(); + return _internal_model_prefix(); } -inline void TrainerSpec::set_model_prefix(const ::std::string& value) { - set_has_model_prefix(); - model_prefix_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void TrainerSpec::set_model_prefix(const std::string& value) { + _internal_set_model_prefix(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.model_prefix) } -#if LANG_CXX11 -inline void TrainerSpec::set_model_prefix(::std::string&& value) { - set_has_model_prefix(); - model_prefix_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_model_prefix() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.model_prefix) + return _internal_mutable_model_prefix(); +} +inline const std::string& TrainerSpec::_internal_model_prefix() const { + return model_prefix_.Get(); +} +inline void TrainerSpec::_internal_set_model_prefix(const std::string& value) { + _has_bits_[0] |= 0x00000001u; + model_prefix_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void TrainerSpec::set_model_prefix(std::string&& value) { + _has_bits_[0] |= 0x00000001u; + model_prefix_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.model_prefix) } -#endif inline void TrainerSpec::set_model_prefix(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_model_prefix(); - model_prefix_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000001u; + model_prefix_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.model_prefix) } -inline void TrainerSpec::set_model_prefix(const char* value, size_t size) { - set_has_model_prefix(); - model_prefix_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_model_prefix(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000001u; + model_prefix_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.model_prefix) } -inline ::std::string* TrainerSpec::mutable_model_prefix() { - set_has_model_prefix(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.model_prefix) - return model_prefix_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* TrainerSpec::_internal_mutable_model_prefix() { + _has_bits_[0] |= 0x00000001u; + return model_prefix_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* TrainerSpec::release_model_prefix() { +inline std::string* TrainerSpec::release_model_prefix() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.model_prefix) - if (!has_model_prefix()) { - return NULL; + if (!_internal_has_model_prefix()) { + return nullptr; } - clear_has_model_prefix(); - return model_prefix_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000001u; + return model_prefix_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void TrainerSpec::set_allocated_model_prefix(::std::string* model_prefix) { - if (model_prefix != NULL) { - set_has_model_prefix(); +inline void TrainerSpec::set_allocated_model_prefix(std::string* model_prefix) { + if (model_prefix != nullptr) { + _has_bits_[0] |= 0x00000001u; } else { - clear_has_model_prefix(); + _has_bits_[0] &= ~0x00000001u; } - model_prefix_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), model_prefix); + model_prefix_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), model_prefix, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.model_prefix) } +inline std::string* TrainerSpec::unsafe_arena_release_model_prefix() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.model_prefix) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000001u; + return model_prefix_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_model_prefix( + std::string* model_prefix) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (model_prefix != nullptr) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + model_prefix_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + model_prefix, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.model_prefix) +} // optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default = UNIGRAM]; -inline bool TrainerSpec::has_model_type() const { - return (_has_bits_[0] & 0x00040000u) != 0; -} -inline void TrainerSpec::set_has_model_type() { - _has_bits_[0] |= 0x00040000u; +inline bool TrainerSpec::_internal_has_model_type() const { + bool value = (_has_bits_[0] & 0x00040000u) != 0; + return value; } -inline void TrainerSpec::clear_has_model_type() { - _has_bits_[0] &= ~0x00040000u; +inline bool TrainerSpec::has_model_type() const { + return _internal_has_model_type(); } inline void TrainerSpec::clear_model_type() { model_type_ = 1; - clear_has_model_type(); + _has_bits_[0] &= ~0x00040000u; +} +inline ::sentencepiece::TrainerSpec_ModelType TrainerSpec::_internal_model_type() const { + return static_cast< ::sentencepiece::TrainerSpec_ModelType >(model_type_); } inline ::sentencepiece::TrainerSpec_ModelType TrainerSpec::model_type() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.model_type) - return static_cast< ::sentencepiece::TrainerSpec_ModelType >(model_type_); + return _internal_model_type(); } -inline void TrainerSpec::set_model_type(::sentencepiece::TrainerSpec_ModelType value) { +inline void TrainerSpec::_internal_set_model_type(::sentencepiece::TrainerSpec_ModelType value) { assert(::sentencepiece::TrainerSpec_ModelType_IsValid(value)); - set_has_model_type(); + _has_bits_[0] |= 0x00040000u; model_type_ = value; +} +inline void TrainerSpec::set_model_type(::sentencepiece::TrainerSpec_ModelType value) { + _internal_set_model_type(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.model_type) } // optional int32 vocab_size = 4 [default = 8000]; -inline bool TrainerSpec::has_vocab_size() const { - return (_has_bits_[0] & 0x00080000u) != 0; +inline bool TrainerSpec::_internal_has_vocab_size() const { + bool value = (_has_bits_[0] & 0x00080000u) != 0; + return value; } -inline void TrainerSpec::set_has_vocab_size() { - _has_bits_[0] |= 0x00080000u; -} -inline void TrainerSpec::clear_has_vocab_size() { - _has_bits_[0] &= ~0x00080000u; +inline bool TrainerSpec::has_vocab_size() const { + return _internal_has_vocab_size(); } inline void TrainerSpec::clear_vocab_size() { vocab_size_ = 8000; - clear_has_vocab_size(); + _has_bits_[0] &= ~0x00080000u; } -inline ::google::protobuf::int32 TrainerSpec::vocab_size() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.vocab_size) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_vocab_size() const { return vocab_size_; } -inline void TrainerSpec::set_vocab_size(::google::protobuf::int32 value) { - set_has_vocab_size(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::vocab_size() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.vocab_size) + return _internal_vocab_size(); +} +inline void TrainerSpec::_internal_set_vocab_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00080000u; vocab_size_ = value; +} +inline void TrainerSpec::set_vocab_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_vocab_size(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.vocab_size) } // repeated string accept_language = 5; -inline int TrainerSpec::accept_language_size() const { +inline int TrainerSpec::_internal_accept_language_size() const { return accept_language_.size(); } +inline int TrainerSpec::accept_language_size() const { + return _internal_accept_language_size(); +} inline void TrainerSpec::clear_accept_language() { accept_language_.Clear(); } -inline const ::std::string& TrainerSpec::accept_language(int index) const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.accept_language) +inline std::string* TrainerSpec::add_accept_language() { + // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.accept_language) + return _internal_add_accept_language(); +} +inline const std::string& TrainerSpec::_internal_accept_language(int index) const { return accept_language_.Get(index); } -inline ::std::string* TrainerSpec::mutable_accept_language(int index) { +inline const std::string& TrainerSpec::accept_language(int index) const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.accept_language) + return _internal_accept_language(index); +} +inline std::string* TrainerSpec::mutable_accept_language(int index) { // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.accept_language) return accept_language_.Mutable(index); } -inline void TrainerSpec::set_accept_language(int index, const ::std::string& value) { +inline void TrainerSpec::set_accept_language(int index, const std::string& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.accept_language) accept_language_.Mutable(index)->assign(value); } -#if LANG_CXX11 -inline void TrainerSpec::set_accept_language(int index, ::std::string&& value) { +inline void TrainerSpec::set_accept_language(int index, std::string&& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.accept_language) accept_language_.Mutable(index)->assign(std::move(value)); } -#endif inline void TrainerSpec::set_accept_language(int index, const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); accept_language_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.accept_language) } @@ -1844,22 +2461,19 @@ inline void TrainerSpec::set_accept_language(int index, const char* value, size_ reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.accept_language) } -inline ::std::string* TrainerSpec::add_accept_language() { - // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.accept_language) +inline std::string* TrainerSpec::_internal_add_accept_language() { return accept_language_.Add(); } -inline void TrainerSpec::add_accept_language(const ::std::string& value) { +inline void TrainerSpec::add_accept_language(const std::string& value) { accept_language_.Add()->assign(value); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.accept_language) } -#if LANG_CXX11 -inline void TrainerSpec::add_accept_language(::std::string&& value) { +inline void TrainerSpec::add_accept_language(std::string&& value) { accept_language_.Add(std::move(value)); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.accept_language) } -#endif inline void TrainerSpec::add_accept_language(const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); accept_language_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:sentencepiece.TrainerSpec.accept_language) } @@ -1867,452 +2481,528 @@ inline void TrainerSpec::add_accept_language(const char* value, size_t size) { accept_language_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:sentencepiece.TrainerSpec.accept_language) } -inline const ::google::protobuf::RepeatedPtrField< ::std::string>& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& TrainerSpec::accept_language() const { // @@protoc_insertion_point(field_list:sentencepiece.TrainerSpec.accept_language) return accept_language_; } -inline ::google::protobuf::RepeatedPtrField< ::std::string>* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* TrainerSpec::mutable_accept_language() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.TrainerSpec.accept_language) return &accept_language_; } // optional int32 self_test_sample_size = 6 [default = 0]; -inline bool TrainerSpec::has_self_test_sample_size() const { - return (_has_bits_[0] & 0x00000100u) != 0; -} -inline void TrainerSpec::set_has_self_test_sample_size() { - _has_bits_[0] |= 0x00000100u; +inline bool TrainerSpec::_internal_has_self_test_sample_size() const { + bool value = (_has_bits_[0] & 0x00000100u) != 0; + return value; } -inline void TrainerSpec::clear_has_self_test_sample_size() { - _has_bits_[0] &= ~0x00000100u; +inline bool TrainerSpec::has_self_test_sample_size() const { + return _internal_has_self_test_sample_size(); } inline void TrainerSpec::clear_self_test_sample_size() { self_test_sample_size_ = 0; - clear_has_self_test_sample_size(); + _has_bits_[0] &= ~0x00000100u; } -inline ::google::protobuf::int32 TrainerSpec::self_test_sample_size() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.self_test_sample_size) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_self_test_sample_size() const { return self_test_sample_size_; } -inline void TrainerSpec::set_self_test_sample_size(::google::protobuf::int32 value) { - set_has_self_test_sample_size(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::self_test_sample_size() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.self_test_sample_size) + return _internal_self_test_sample_size(); +} +inline void TrainerSpec::_internal_set_self_test_sample_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00000100u; self_test_sample_size_ = value; +} +inline void TrainerSpec::set_self_test_sample_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_self_test_sample_size(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.self_test_sample_size) } // optional float character_coverage = 10 [default = 0.9995]; -inline bool TrainerSpec::has_character_coverage() const { - return (_has_bits_[0] & 0x00100000u) != 0; -} -inline void TrainerSpec::set_has_character_coverage() { - _has_bits_[0] |= 0x00100000u; +inline bool TrainerSpec::_internal_has_character_coverage() const { + bool value = (_has_bits_[0] & 0x00100000u) != 0; + return value; } -inline void TrainerSpec::clear_has_character_coverage() { - _has_bits_[0] &= ~0x00100000u; +inline bool TrainerSpec::has_character_coverage() const { + return _internal_has_character_coverage(); } inline void TrainerSpec::clear_character_coverage() { character_coverage_ = 0.9995f; - clear_has_character_coverage(); + _has_bits_[0] &= ~0x00100000u; +} +inline float TrainerSpec::_internal_character_coverage() const { + return character_coverage_; } inline float TrainerSpec::character_coverage() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.character_coverage) - return character_coverage_; + return _internal_character_coverage(); } -inline void TrainerSpec::set_character_coverage(float value) { - set_has_character_coverage(); +inline void TrainerSpec::_internal_set_character_coverage(float value) { + _has_bits_[0] |= 0x00100000u; character_coverage_ = value; +} +inline void TrainerSpec::set_character_coverage(float value) { + _internal_set_character_coverage(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.character_coverage) } // optional int32 input_sentence_size = 11 [default = 0]; -inline bool TrainerSpec::has_input_sentence_size() const { - return (_has_bits_[0] & 0x00000200u) != 0; -} -inline void TrainerSpec::set_has_input_sentence_size() { - _has_bits_[0] |= 0x00000200u; +inline bool TrainerSpec::_internal_has_input_sentence_size() const { + bool value = (_has_bits_[0] & 0x00000200u) != 0; + return value; } -inline void TrainerSpec::clear_has_input_sentence_size() { - _has_bits_[0] &= ~0x00000200u; +inline bool TrainerSpec::has_input_sentence_size() const { + return _internal_has_input_sentence_size(); } inline void TrainerSpec::clear_input_sentence_size() { input_sentence_size_ = 0; - clear_has_input_sentence_size(); + _has_bits_[0] &= ~0x00000200u; } -inline ::google::protobuf::int32 TrainerSpec::input_sentence_size() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.input_sentence_size) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_input_sentence_size() const { return input_sentence_size_; } -inline void TrainerSpec::set_input_sentence_size(::google::protobuf::int32 value) { - set_has_input_sentence_size(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::input_sentence_size() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.input_sentence_size) + return _internal_input_sentence_size(); +} +inline void TrainerSpec::_internal_set_input_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00000200u; input_sentence_size_ = value; +} +inline void TrainerSpec::set_input_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_input_sentence_size(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.input_sentence_size) } // optional bool shuffle_input_sentence = 19 [default = true]; -inline bool TrainerSpec::has_shuffle_input_sentence() const { - return (_has_bits_[0] & 0x08000000u) != 0; -} -inline void TrainerSpec::set_has_shuffle_input_sentence() { - _has_bits_[0] |= 0x08000000u; +inline bool TrainerSpec::_internal_has_shuffle_input_sentence() const { + bool value = (_has_bits_[0] & 0x08000000u) != 0; + return value; } -inline void TrainerSpec::clear_has_shuffle_input_sentence() { - _has_bits_[0] &= ~0x08000000u; +inline bool TrainerSpec::has_shuffle_input_sentence() const { + return _internal_has_shuffle_input_sentence(); } inline void TrainerSpec::clear_shuffle_input_sentence() { shuffle_input_sentence_ = true; - clear_has_shuffle_input_sentence(); + _has_bits_[0] &= ~0x08000000u; +} +inline bool TrainerSpec::_internal_shuffle_input_sentence() const { + return shuffle_input_sentence_; } inline bool TrainerSpec::shuffle_input_sentence() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.shuffle_input_sentence) - return shuffle_input_sentence_; + return _internal_shuffle_input_sentence(); } -inline void TrainerSpec::set_shuffle_input_sentence(bool value) { - set_has_shuffle_input_sentence(); +inline void TrainerSpec::_internal_set_shuffle_input_sentence(bool value) { + _has_bits_[0] |= 0x08000000u; shuffle_input_sentence_ = value; +} +inline void TrainerSpec::set_shuffle_input_sentence(bool value) { + _internal_set_shuffle_input_sentence(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.shuffle_input_sentence) } // optional int32 mining_sentence_size = 12 [deprecated = true]; -inline bool TrainerSpec::has_mining_sentence_size() const { - return (_has_bits_[0] & 0x00000400u) != 0; -} -inline void TrainerSpec::set_has_mining_sentence_size() { - _has_bits_[0] |= 0x00000400u; +inline bool TrainerSpec::_internal_has_mining_sentence_size() const { + bool value = (_has_bits_[0] & 0x00000400u) != 0; + return value; } -inline void TrainerSpec::clear_has_mining_sentence_size() { - _has_bits_[0] &= ~0x00000400u; +inline bool TrainerSpec::has_mining_sentence_size() const { + return _internal_has_mining_sentence_size(); } inline void TrainerSpec::clear_mining_sentence_size() { mining_sentence_size_ = 0; - clear_has_mining_sentence_size(); + _has_bits_[0] &= ~0x00000400u; } -inline ::google::protobuf::int32 TrainerSpec::mining_sentence_size() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.mining_sentence_size) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_mining_sentence_size() const { return mining_sentence_size_; } -inline void TrainerSpec::set_mining_sentence_size(::google::protobuf::int32 value) { - set_has_mining_sentence_size(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::mining_sentence_size() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.mining_sentence_size) + return _internal_mining_sentence_size(); +} +inline void TrainerSpec::_internal_set_mining_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00000400u; mining_sentence_size_ = value; +} +inline void TrainerSpec::set_mining_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_mining_sentence_size(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.mining_sentence_size) } // optional int32 training_sentence_size = 13 [deprecated = true]; -inline bool TrainerSpec::has_training_sentence_size() const { - return (_has_bits_[0] & 0x00000800u) != 0; +inline bool TrainerSpec::_internal_has_training_sentence_size() const { + bool value = (_has_bits_[0] & 0x00000800u) != 0; + return value; } -inline void TrainerSpec::set_has_training_sentence_size() { - _has_bits_[0] |= 0x00000800u; -} -inline void TrainerSpec::clear_has_training_sentence_size() { - _has_bits_[0] &= ~0x00000800u; +inline bool TrainerSpec::has_training_sentence_size() const { + return _internal_has_training_sentence_size(); } inline void TrainerSpec::clear_training_sentence_size() { training_sentence_size_ = 0; - clear_has_training_sentence_size(); + _has_bits_[0] &= ~0x00000800u; } -inline ::google::protobuf::int32 TrainerSpec::training_sentence_size() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.training_sentence_size) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_training_sentence_size() const { return training_sentence_size_; } -inline void TrainerSpec::set_training_sentence_size(::google::protobuf::int32 value) { - set_has_training_sentence_size(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::training_sentence_size() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.training_sentence_size) + return _internal_training_sentence_size(); +} +inline void TrainerSpec::_internal_set_training_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00000800u; training_sentence_size_ = value; +} +inline void TrainerSpec::set_training_sentence_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_training_sentence_size(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.training_sentence_size) } // optional int32 seed_sentencepiece_size = 14 [default = 1000000]; -inline bool TrainerSpec::has_seed_sentencepiece_size() const { - return (_has_bits_[0] & 0x00200000u) != 0; +inline bool TrainerSpec::_internal_has_seed_sentencepiece_size() const { + bool value = (_has_bits_[0] & 0x00200000u) != 0; + return value; } -inline void TrainerSpec::set_has_seed_sentencepiece_size() { - _has_bits_[0] |= 0x00200000u; -} -inline void TrainerSpec::clear_has_seed_sentencepiece_size() { - _has_bits_[0] &= ~0x00200000u; +inline bool TrainerSpec::has_seed_sentencepiece_size() const { + return _internal_has_seed_sentencepiece_size(); } inline void TrainerSpec::clear_seed_sentencepiece_size() { seed_sentencepiece_size_ = 1000000; - clear_has_seed_sentencepiece_size(); + _has_bits_[0] &= ~0x00200000u; } -inline ::google::protobuf::int32 TrainerSpec::seed_sentencepiece_size() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.seed_sentencepiece_size) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_seed_sentencepiece_size() const { return seed_sentencepiece_size_; } -inline void TrainerSpec::set_seed_sentencepiece_size(::google::protobuf::int32 value) { - set_has_seed_sentencepiece_size(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::seed_sentencepiece_size() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.seed_sentencepiece_size) + return _internal_seed_sentencepiece_size(); +} +inline void TrainerSpec::_internal_set_seed_sentencepiece_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00200000u; seed_sentencepiece_size_ = value; +} +inline void TrainerSpec::set_seed_sentencepiece_size(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_seed_sentencepiece_size(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.seed_sentencepiece_size) } // optional float shrinking_factor = 15 [default = 0.75]; -inline bool TrainerSpec::has_shrinking_factor() const { - return (_has_bits_[0] & 0x00400000u) != 0; -} -inline void TrainerSpec::set_has_shrinking_factor() { - _has_bits_[0] |= 0x00400000u; +inline bool TrainerSpec::_internal_has_shrinking_factor() const { + bool value = (_has_bits_[0] & 0x00400000u) != 0; + return value; } -inline void TrainerSpec::clear_has_shrinking_factor() { - _has_bits_[0] &= ~0x00400000u; +inline bool TrainerSpec::has_shrinking_factor() const { + return _internal_has_shrinking_factor(); } inline void TrainerSpec::clear_shrinking_factor() { shrinking_factor_ = 0.75f; - clear_has_shrinking_factor(); + _has_bits_[0] &= ~0x00400000u; +} +inline float TrainerSpec::_internal_shrinking_factor() const { + return shrinking_factor_; } inline float TrainerSpec::shrinking_factor() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.shrinking_factor) - return shrinking_factor_; + return _internal_shrinking_factor(); } -inline void TrainerSpec::set_shrinking_factor(float value) { - set_has_shrinking_factor(); +inline void TrainerSpec::_internal_set_shrinking_factor(float value) { + _has_bits_[0] |= 0x00400000u; shrinking_factor_ = value; +} +inline void TrainerSpec::set_shrinking_factor(float value) { + _internal_set_shrinking_factor(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.shrinking_factor) } // optional int32 max_sentence_length = 18 [default = 4192]; -inline bool TrainerSpec::has_max_sentence_length() const { - return (_has_bits_[0] & 0x02000000u) != 0; +inline bool TrainerSpec::_internal_has_max_sentence_length() const { + bool value = (_has_bits_[0] & 0x02000000u) != 0; + return value; } -inline void TrainerSpec::set_has_max_sentence_length() { - _has_bits_[0] |= 0x02000000u; -} -inline void TrainerSpec::clear_has_max_sentence_length() { - _has_bits_[0] &= ~0x02000000u; +inline bool TrainerSpec::has_max_sentence_length() const { + return _internal_has_max_sentence_length(); } inline void TrainerSpec::clear_max_sentence_length() { max_sentence_length_ = 4192; - clear_has_max_sentence_length(); + _has_bits_[0] &= ~0x02000000u; } -inline ::google::protobuf::int32 TrainerSpec::max_sentence_length() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.max_sentence_length) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_max_sentence_length() const { return max_sentence_length_; } -inline void TrainerSpec::set_max_sentence_length(::google::protobuf::int32 value) { - set_has_max_sentence_length(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::max_sentence_length() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.max_sentence_length) + return _internal_max_sentence_length(); +} +inline void TrainerSpec::_internal_set_max_sentence_length(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x02000000u; max_sentence_length_ = value; +} +inline void TrainerSpec::set_max_sentence_length(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_max_sentence_length(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.max_sentence_length) } // optional int32 num_threads = 16 [default = 16]; -inline bool TrainerSpec::has_num_threads() const { - return (_has_bits_[0] & 0x00800000u) != 0; -} -inline void TrainerSpec::set_has_num_threads() { - _has_bits_[0] |= 0x00800000u; +inline bool TrainerSpec::_internal_has_num_threads() const { + bool value = (_has_bits_[0] & 0x00800000u) != 0; + return value; } -inline void TrainerSpec::clear_has_num_threads() { - _has_bits_[0] &= ~0x00800000u; +inline bool TrainerSpec::has_num_threads() const { + return _internal_has_num_threads(); } inline void TrainerSpec::clear_num_threads() { num_threads_ = 16; - clear_has_num_threads(); + _has_bits_[0] &= ~0x00800000u; } -inline ::google::protobuf::int32 TrainerSpec::num_threads() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.num_threads) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_num_threads() const { return num_threads_; } -inline void TrainerSpec::set_num_threads(::google::protobuf::int32 value) { - set_has_num_threads(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::num_threads() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.num_threads) + return _internal_num_threads(); +} +inline void TrainerSpec::_internal_set_num_threads(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00800000u; num_threads_ = value; +} +inline void TrainerSpec::set_num_threads(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_num_threads(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.num_threads) } // optional int32 num_sub_iterations = 17 [default = 2]; -inline bool TrainerSpec::has_num_sub_iterations() const { - return (_has_bits_[0] & 0x01000000u) != 0; -} -inline void TrainerSpec::set_has_num_sub_iterations() { - _has_bits_[0] |= 0x01000000u; +inline bool TrainerSpec::_internal_has_num_sub_iterations() const { + bool value = (_has_bits_[0] & 0x01000000u) != 0; + return value; } -inline void TrainerSpec::clear_has_num_sub_iterations() { - _has_bits_[0] &= ~0x01000000u; +inline bool TrainerSpec::has_num_sub_iterations() const { + return _internal_has_num_sub_iterations(); } inline void TrainerSpec::clear_num_sub_iterations() { num_sub_iterations_ = 2; - clear_has_num_sub_iterations(); + _has_bits_[0] &= ~0x01000000u; } -inline ::google::protobuf::int32 TrainerSpec::num_sub_iterations() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.num_sub_iterations) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_num_sub_iterations() const { return num_sub_iterations_; } -inline void TrainerSpec::set_num_sub_iterations(::google::protobuf::int32 value) { - set_has_num_sub_iterations(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::num_sub_iterations() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.num_sub_iterations) + return _internal_num_sub_iterations(); +} +inline void TrainerSpec::_internal_set_num_sub_iterations(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x01000000u; num_sub_iterations_ = value; +} +inline void TrainerSpec::set_num_sub_iterations(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_num_sub_iterations(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.num_sub_iterations) } // optional int32 max_sentencepiece_length = 20 [default = 16]; -inline bool TrainerSpec::has_max_sentencepiece_length() const { - return (_has_bits_[0] & 0x04000000u) != 0; -} -inline void TrainerSpec::set_has_max_sentencepiece_length() { - _has_bits_[0] |= 0x04000000u; +inline bool TrainerSpec::_internal_has_max_sentencepiece_length() const { + bool value = (_has_bits_[0] & 0x04000000u) != 0; + return value; } -inline void TrainerSpec::clear_has_max_sentencepiece_length() { - _has_bits_[0] &= ~0x04000000u; +inline bool TrainerSpec::has_max_sentencepiece_length() const { + return _internal_has_max_sentencepiece_length(); } inline void TrainerSpec::clear_max_sentencepiece_length() { max_sentencepiece_length_ = 16; - clear_has_max_sentencepiece_length(); + _has_bits_[0] &= ~0x04000000u; } -inline ::google::protobuf::int32 TrainerSpec::max_sentencepiece_length() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.max_sentencepiece_length) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_max_sentencepiece_length() const { return max_sentencepiece_length_; } -inline void TrainerSpec::set_max_sentencepiece_length(::google::protobuf::int32 value) { - set_has_max_sentencepiece_length(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::max_sentencepiece_length() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.max_sentencepiece_length) + return _internal_max_sentencepiece_length(); +} +inline void TrainerSpec::_internal_set_max_sentencepiece_length(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x04000000u; max_sentencepiece_length_ = value; +} +inline void TrainerSpec::set_max_sentencepiece_length(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_max_sentencepiece_length(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.max_sentencepiece_length) } // optional bool split_by_unicode_script = 21 [default = true]; -inline bool TrainerSpec::has_split_by_unicode_script() const { - return (_has_bits_[0] & 0x10000000u) != 0; +inline bool TrainerSpec::_internal_has_split_by_unicode_script() const { + bool value = (_has_bits_[0] & 0x10000000u) != 0; + return value; } -inline void TrainerSpec::set_has_split_by_unicode_script() { - _has_bits_[0] |= 0x10000000u; -} -inline void TrainerSpec::clear_has_split_by_unicode_script() { - _has_bits_[0] &= ~0x10000000u; +inline bool TrainerSpec::has_split_by_unicode_script() const { + return _internal_has_split_by_unicode_script(); } inline void TrainerSpec::clear_split_by_unicode_script() { split_by_unicode_script_ = true; - clear_has_split_by_unicode_script(); + _has_bits_[0] &= ~0x10000000u; +} +inline bool TrainerSpec::_internal_split_by_unicode_script() const { + return split_by_unicode_script_; } inline bool TrainerSpec::split_by_unicode_script() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.split_by_unicode_script) - return split_by_unicode_script_; + return _internal_split_by_unicode_script(); } -inline void TrainerSpec::set_split_by_unicode_script(bool value) { - set_has_split_by_unicode_script(); +inline void TrainerSpec::_internal_set_split_by_unicode_script(bool value) { + _has_bits_[0] |= 0x10000000u; split_by_unicode_script_ = value; +} +inline void TrainerSpec::set_split_by_unicode_script(bool value) { + _internal_set_split_by_unicode_script(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.split_by_unicode_script) } // optional bool split_by_number = 23 [default = true]; -inline bool TrainerSpec::has_split_by_number() const { - return (_has_bits_[0] & 0x20000000u) != 0; -} -inline void TrainerSpec::set_has_split_by_number() { - _has_bits_[0] |= 0x20000000u; +inline bool TrainerSpec::_internal_has_split_by_number() const { + bool value = (_has_bits_[0] & 0x20000000u) != 0; + return value; } -inline void TrainerSpec::clear_has_split_by_number() { - _has_bits_[0] &= ~0x20000000u; +inline bool TrainerSpec::has_split_by_number() const { + return _internal_has_split_by_number(); } inline void TrainerSpec::clear_split_by_number() { split_by_number_ = true; - clear_has_split_by_number(); + _has_bits_[0] &= ~0x20000000u; +} +inline bool TrainerSpec::_internal_split_by_number() const { + return split_by_number_; } inline bool TrainerSpec::split_by_number() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.split_by_number) - return split_by_number_; + return _internal_split_by_number(); } -inline void TrainerSpec::set_split_by_number(bool value) { - set_has_split_by_number(); +inline void TrainerSpec::_internal_set_split_by_number(bool value) { + _has_bits_[0] |= 0x20000000u; split_by_number_ = value; +} +inline void TrainerSpec::set_split_by_number(bool value) { + _internal_set_split_by_number(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.split_by_number) } // optional bool split_by_whitespace = 22 [default = true]; -inline bool TrainerSpec::has_split_by_whitespace() const { - return (_has_bits_[0] & 0x40000000u) != 0; -} -inline void TrainerSpec::set_has_split_by_whitespace() { - _has_bits_[0] |= 0x40000000u; +inline bool TrainerSpec::_internal_has_split_by_whitespace() const { + bool value = (_has_bits_[0] & 0x40000000u) != 0; + return value; } -inline void TrainerSpec::clear_has_split_by_whitespace() { - _has_bits_[0] &= ~0x40000000u; +inline bool TrainerSpec::has_split_by_whitespace() const { + return _internal_has_split_by_whitespace(); } inline void TrainerSpec::clear_split_by_whitespace() { split_by_whitespace_ = true; - clear_has_split_by_whitespace(); + _has_bits_[0] &= ~0x40000000u; +} +inline bool TrainerSpec::_internal_split_by_whitespace() const { + return split_by_whitespace_; } inline bool TrainerSpec::split_by_whitespace() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.split_by_whitespace) - return split_by_whitespace_; + return _internal_split_by_whitespace(); } -inline void TrainerSpec::set_split_by_whitespace(bool value) { - set_has_split_by_whitespace(); +inline void TrainerSpec::_internal_set_split_by_whitespace(bool value) { + _has_bits_[0] |= 0x40000000u; split_by_whitespace_ = value; +} +inline void TrainerSpec::set_split_by_whitespace(bool value) { + _internal_set_split_by_whitespace(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.split_by_whitespace) } // optional bool treat_whitespace_as_suffix = 24 [default = false]; -inline bool TrainerSpec::has_treat_whitespace_as_suffix() const { - return (_has_bits_[0] & 0x00001000u) != 0; -} -inline void TrainerSpec::set_has_treat_whitespace_as_suffix() { - _has_bits_[0] |= 0x00001000u; +inline bool TrainerSpec::_internal_has_treat_whitespace_as_suffix() const { + bool value = (_has_bits_[0] & 0x00001000u) != 0; + return value; } -inline void TrainerSpec::clear_has_treat_whitespace_as_suffix() { - _has_bits_[0] &= ~0x00001000u; +inline bool TrainerSpec::has_treat_whitespace_as_suffix() const { + return _internal_has_treat_whitespace_as_suffix(); } inline void TrainerSpec::clear_treat_whitespace_as_suffix() { treat_whitespace_as_suffix_ = false; - clear_has_treat_whitespace_as_suffix(); + _has_bits_[0] &= ~0x00001000u; +} +inline bool TrainerSpec::_internal_treat_whitespace_as_suffix() const { + return treat_whitespace_as_suffix_; } inline bool TrainerSpec::treat_whitespace_as_suffix() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.treat_whitespace_as_suffix) - return treat_whitespace_as_suffix_; + return _internal_treat_whitespace_as_suffix(); } -inline void TrainerSpec::set_treat_whitespace_as_suffix(bool value) { - set_has_treat_whitespace_as_suffix(); +inline void TrainerSpec::_internal_set_treat_whitespace_as_suffix(bool value) { + _has_bits_[0] |= 0x00001000u; treat_whitespace_as_suffix_ = value; +} +inline void TrainerSpec::set_treat_whitespace_as_suffix(bool value) { + _internal_set_treat_whitespace_as_suffix(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.treat_whitespace_as_suffix) } // optional bool split_digits = 25 [default = false]; -inline bool TrainerSpec::has_split_digits() const { - return (_has_bits_[0] & 0x00002000u) != 0; +inline bool TrainerSpec::_internal_has_split_digits() const { + bool value = (_has_bits_[0] & 0x00002000u) != 0; + return value; } -inline void TrainerSpec::set_has_split_digits() { - _has_bits_[0] |= 0x00002000u; -} -inline void TrainerSpec::clear_has_split_digits() { - _has_bits_[0] &= ~0x00002000u; +inline bool TrainerSpec::has_split_digits() const { + return _internal_has_split_digits(); } inline void TrainerSpec::clear_split_digits() { split_digits_ = false; - clear_has_split_digits(); + _has_bits_[0] &= ~0x00002000u; +} +inline bool TrainerSpec::_internal_split_digits() const { + return split_digits_; } inline bool TrainerSpec::split_digits() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.split_digits) - return split_digits_; + return _internal_split_digits(); } -inline void TrainerSpec::set_split_digits(bool value) { - set_has_split_digits(); +inline void TrainerSpec::_internal_set_split_digits(bool value) { + _has_bits_[0] |= 0x00002000u; split_digits_ = value; +} +inline void TrainerSpec::set_split_digits(bool value) { + _internal_set_split_digits(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.split_digits) } // repeated string control_symbols = 30; -inline int TrainerSpec::control_symbols_size() const { +inline int TrainerSpec::_internal_control_symbols_size() const { return control_symbols_.size(); } +inline int TrainerSpec::control_symbols_size() const { + return _internal_control_symbols_size(); +} inline void TrainerSpec::clear_control_symbols() { control_symbols_.Clear(); } -inline const ::std::string& TrainerSpec::control_symbols(int index) const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.control_symbols) +inline std::string* TrainerSpec::add_control_symbols() { + // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.control_symbols) + return _internal_add_control_symbols(); +} +inline const std::string& TrainerSpec::_internal_control_symbols(int index) const { return control_symbols_.Get(index); } -inline ::std::string* TrainerSpec::mutable_control_symbols(int index) { +inline const std::string& TrainerSpec::control_symbols(int index) const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.control_symbols) + return _internal_control_symbols(index); +} +inline std::string* TrainerSpec::mutable_control_symbols(int index) { // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.control_symbols) return control_symbols_.Mutable(index); } -inline void TrainerSpec::set_control_symbols(int index, const ::std::string& value) { +inline void TrainerSpec::set_control_symbols(int index, const std::string& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.control_symbols) control_symbols_.Mutable(index)->assign(value); } -#if LANG_CXX11 -inline void TrainerSpec::set_control_symbols(int index, ::std::string&& value) { +inline void TrainerSpec::set_control_symbols(int index, std::string&& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.control_symbols) control_symbols_.Mutable(index)->assign(std::move(value)); } -#endif inline void TrainerSpec::set_control_symbols(int index, const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); control_symbols_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.control_symbols) } @@ -2321,22 +3011,19 @@ inline void TrainerSpec::set_control_symbols(int index, const char* value, size_ reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.control_symbols) } -inline ::std::string* TrainerSpec::add_control_symbols() { - // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.control_symbols) +inline std::string* TrainerSpec::_internal_add_control_symbols() { return control_symbols_.Add(); } -inline void TrainerSpec::add_control_symbols(const ::std::string& value) { +inline void TrainerSpec::add_control_symbols(const std::string& value) { control_symbols_.Add()->assign(value); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.control_symbols) } -#if LANG_CXX11 -inline void TrainerSpec::add_control_symbols(::std::string&& value) { +inline void TrainerSpec::add_control_symbols(std::string&& value) { control_symbols_.Add(std::move(value)); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.control_symbols) } -#endif inline void TrainerSpec::add_control_symbols(const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); control_symbols_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:sentencepiece.TrainerSpec.control_symbols) } @@ -2344,44 +3031,52 @@ inline void TrainerSpec::add_control_symbols(const char* value, size_t size) { control_symbols_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:sentencepiece.TrainerSpec.control_symbols) } -inline const ::google::protobuf::RepeatedPtrField< ::std::string>& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& TrainerSpec::control_symbols() const { // @@protoc_insertion_point(field_list:sentencepiece.TrainerSpec.control_symbols) return control_symbols_; } -inline ::google::protobuf::RepeatedPtrField< ::std::string>* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* TrainerSpec::mutable_control_symbols() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.TrainerSpec.control_symbols) return &control_symbols_; } // repeated string user_defined_symbols = 31; -inline int TrainerSpec::user_defined_symbols_size() const { +inline int TrainerSpec::_internal_user_defined_symbols_size() const { return user_defined_symbols_.size(); } +inline int TrainerSpec::user_defined_symbols_size() const { + return _internal_user_defined_symbols_size(); +} inline void TrainerSpec::clear_user_defined_symbols() { user_defined_symbols_.Clear(); } -inline const ::std::string& TrainerSpec::user_defined_symbols(int index) const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.user_defined_symbols) +inline std::string* TrainerSpec::add_user_defined_symbols() { + // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.user_defined_symbols) + return _internal_add_user_defined_symbols(); +} +inline const std::string& TrainerSpec::_internal_user_defined_symbols(int index) const { return user_defined_symbols_.Get(index); } -inline ::std::string* TrainerSpec::mutable_user_defined_symbols(int index) { +inline const std::string& TrainerSpec::user_defined_symbols(int index) const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.user_defined_symbols) + return _internal_user_defined_symbols(index); +} +inline std::string* TrainerSpec::mutable_user_defined_symbols(int index) { // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.user_defined_symbols) return user_defined_symbols_.Mutable(index); } -inline void TrainerSpec::set_user_defined_symbols(int index, const ::std::string& value) { +inline void TrainerSpec::set_user_defined_symbols(int index, const std::string& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.user_defined_symbols) user_defined_symbols_.Mutable(index)->assign(value); } -#if LANG_CXX11 -inline void TrainerSpec::set_user_defined_symbols(int index, ::std::string&& value) { +inline void TrainerSpec::set_user_defined_symbols(int index, std::string&& value) { // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.user_defined_symbols) user_defined_symbols_.Mutable(index)->assign(std::move(value)); } -#endif inline void TrainerSpec::set_user_defined_symbols(int index, const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); user_defined_symbols_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.user_defined_symbols) } @@ -2390,22 +3085,19 @@ inline void TrainerSpec::set_user_defined_symbols(int index, const char* value, reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.user_defined_symbols) } -inline ::std::string* TrainerSpec::add_user_defined_symbols() { - // @@protoc_insertion_point(field_add_mutable:sentencepiece.TrainerSpec.user_defined_symbols) +inline std::string* TrainerSpec::_internal_add_user_defined_symbols() { return user_defined_symbols_.Add(); } -inline void TrainerSpec::add_user_defined_symbols(const ::std::string& value) { +inline void TrainerSpec::add_user_defined_symbols(const std::string& value) { user_defined_symbols_.Add()->assign(value); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.user_defined_symbols) } -#if LANG_CXX11 -inline void TrainerSpec::add_user_defined_symbols(::std::string&& value) { +inline void TrainerSpec::add_user_defined_symbols(std::string&& value) { user_defined_symbols_.Add(std::move(value)); // @@protoc_insertion_point(field_add:sentencepiece.TrainerSpec.user_defined_symbols) } -#endif inline void TrainerSpec::add_user_defined_symbols(const char* value) { - GOOGLE_DCHECK(value != NULL); + GOOGLE_DCHECK(value != nullptr); user_defined_symbols_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:sentencepiece.TrainerSpec.user_defined_symbols) } @@ -2413,626 +3105,824 @@ inline void TrainerSpec::add_user_defined_symbols(const char* value, size_t size user_defined_symbols_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:sentencepiece.TrainerSpec.user_defined_symbols) } -inline const ::google::protobuf::RepeatedPtrField< ::std::string>& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& TrainerSpec::user_defined_symbols() const { // @@protoc_insertion_point(field_list:sentencepiece.TrainerSpec.user_defined_symbols) return user_defined_symbols_; } -inline ::google::protobuf::RepeatedPtrField< ::std::string>* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* TrainerSpec::mutable_user_defined_symbols() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.TrainerSpec.user_defined_symbols) return &user_defined_symbols_; } // optional string required_chars = 36; -inline bool TrainerSpec::has_required_chars() const { - return (_has_bits_[0] & 0x00000004u) != 0; -} -inline void TrainerSpec::set_has_required_chars() { - _has_bits_[0] |= 0x00000004u; +inline bool TrainerSpec::_internal_has_required_chars() const { + bool value = (_has_bits_[0] & 0x00000004u) != 0; + return value; } -inline void TrainerSpec::clear_has_required_chars() { - _has_bits_[0] &= ~0x00000004u; +inline bool TrainerSpec::has_required_chars() const { + return _internal_has_required_chars(); } inline void TrainerSpec::clear_required_chars() { - required_chars_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_required_chars(); + required_chars_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000004u; } -inline const ::std::string& TrainerSpec::required_chars() const { +inline const std::string& TrainerSpec::required_chars() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.required_chars) - return required_chars_.GetNoArena(); + return _internal_required_chars(); } -inline void TrainerSpec::set_required_chars(const ::std::string& value) { - set_has_required_chars(); - required_chars_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void TrainerSpec::set_required_chars(const std::string& value) { + _internal_set_required_chars(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.required_chars) } -#if LANG_CXX11 -inline void TrainerSpec::set_required_chars(::std::string&& value) { - set_has_required_chars(); - required_chars_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_required_chars() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.required_chars) + return _internal_mutable_required_chars(); +} +inline const std::string& TrainerSpec::_internal_required_chars() const { + return required_chars_.Get(); +} +inline void TrainerSpec::_internal_set_required_chars(const std::string& value) { + _has_bits_[0] |= 0x00000004u; + required_chars_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void TrainerSpec::set_required_chars(std::string&& value) { + _has_bits_[0] |= 0x00000004u; + required_chars_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.required_chars) } -#endif inline void TrainerSpec::set_required_chars(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_required_chars(); - required_chars_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000004u; + required_chars_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.required_chars) } -inline void TrainerSpec::set_required_chars(const char* value, size_t size) { - set_has_required_chars(); - required_chars_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_required_chars(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000004u; + required_chars_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.required_chars) } -inline ::std::string* TrainerSpec::mutable_required_chars() { - set_has_required_chars(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.required_chars) - return required_chars_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* TrainerSpec::_internal_mutable_required_chars() { + _has_bits_[0] |= 0x00000004u; + return required_chars_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* TrainerSpec::release_required_chars() { +inline std::string* TrainerSpec::release_required_chars() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.required_chars) - if (!has_required_chars()) { - return NULL; + if (!_internal_has_required_chars()) { + return nullptr; } - clear_has_required_chars(); - return required_chars_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000004u; + return required_chars_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void TrainerSpec::set_allocated_required_chars(::std::string* required_chars) { - if (required_chars != NULL) { - set_has_required_chars(); +inline void TrainerSpec::set_allocated_required_chars(std::string* required_chars) { + if (required_chars != nullptr) { + _has_bits_[0] |= 0x00000004u; } else { - clear_has_required_chars(); + _has_bits_[0] &= ~0x00000004u; } - required_chars_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), required_chars); + required_chars_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), required_chars, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.required_chars) } +inline std::string* TrainerSpec::unsafe_arena_release_required_chars() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.required_chars) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000004u; + return required_chars_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_required_chars( + std::string* required_chars) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (required_chars != nullptr) { + _has_bits_[0] |= 0x00000004u; + } else { + _has_bits_[0] &= ~0x00000004u; + } + required_chars_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + required_chars, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.required_chars) +} // optional bool byte_fallback = 35 [default = false]; -inline bool TrainerSpec::has_byte_fallback() const { - return (_has_bits_[0] & 0x00004000u) != 0; -} -inline void TrainerSpec::set_has_byte_fallback() { - _has_bits_[0] |= 0x00004000u; +inline bool TrainerSpec::_internal_has_byte_fallback() const { + bool value = (_has_bits_[0] & 0x00004000u) != 0; + return value; } -inline void TrainerSpec::clear_has_byte_fallback() { - _has_bits_[0] &= ~0x00004000u; +inline bool TrainerSpec::has_byte_fallback() const { + return _internal_has_byte_fallback(); } inline void TrainerSpec::clear_byte_fallback() { byte_fallback_ = false; - clear_has_byte_fallback(); + _has_bits_[0] &= ~0x00004000u; +} +inline bool TrainerSpec::_internal_byte_fallback() const { + return byte_fallback_; } inline bool TrainerSpec::byte_fallback() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.byte_fallback) - return byte_fallback_; + return _internal_byte_fallback(); } -inline void TrainerSpec::set_byte_fallback(bool value) { - set_has_byte_fallback(); +inline void TrainerSpec::_internal_set_byte_fallback(bool value) { + _has_bits_[0] |= 0x00004000u; byte_fallback_ = value; +} +inline void TrainerSpec::set_byte_fallback(bool value) { + _internal_set_byte_fallback(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.byte_fallback) } // optional bool vocabulary_output_piece_score = 32 [default = true]; -inline bool TrainerSpec::has_vocabulary_output_piece_score() const { - return (_has_bits_[0] & 0x80000000u) != 0; -} -inline void TrainerSpec::set_has_vocabulary_output_piece_score() { - _has_bits_[0] |= 0x80000000u; +inline bool TrainerSpec::_internal_has_vocabulary_output_piece_score() const { + bool value = (_has_bits_[0] & 0x80000000u) != 0; + return value; } -inline void TrainerSpec::clear_has_vocabulary_output_piece_score() { - _has_bits_[0] &= ~0x80000000u; +inline bool TrainerSpec::has_vocabulary_output_piece_score() const { + return _internal_has_vocabulary_output_piece_score(); } inline void TrainerSpec::clear_vocabulary_output_piece_score() { vocabulary_output_piece_score_ = true; - clear_has_vocabulary_output_piece_score(); + _has_bits_[0] &= ~0x80000000u; +} +inline bool TrainerSpec::_internal_vocabulary_output_piece_score() const { + return vocabulary_output_piece_score_; } inline bool TrainerSpec::vocabulary_output_piece_score() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.vocabulary_output_piece_score) - return vocabulary_output_piece_score_; + return _internal_vocabulary_output_piece_score(); } -inline void TrainerSpec::set_vocabulary_output_piece_score(bool value) { - set_has_vocabulary_output_piece_score(); +inline void TrainerSpec::_internal_set_vocabulary_output_piece_score(bool value) { + _has_bits_[0] |= 0x80000000u; vocabulary_output_piece_score_ = value; +} +inline void TrainerSpec::set_vocabulary_output_piece_score(bool value) { + _internal_set_vocabulary_output_piece_score(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.vocabulary_output_piece_score) } // optional bool hard_vocab_limit = 33 [default = true]; -inline bool TrainerSpec::has_hard_vocab_limit() const { - return (_has_bits_[1] & 0x00000001u) != 0; -} -inline void TrainerSpec::set_has_hard_vocab_limit() { - _has_bits_[1] |= 0x00000001u; +inline bool TrainerSpec::_internal_has_hard_vocab_limit() const { + bool value = (_has_bits_[1] & 0x00000001u) != 0; + return value; } -inline void TrainerSpec::clear_has_hard_vocab_limit() { - _has_bits_[1] &= ~0x00000001u; +inline bool TrainerSpec::has_hard_vocab_limit() const { + return _internal_has_hard_vocab_limit(); } inline void TrainerSpec::clear_hard_vocab_limit() { hard_vocab_limit_ = true; - clear_has_hard_vocab_limit(); + _has_bits_[1] &= ~0x00000001u; +} +inline bool TrainerSpec::_internal_hard_vocab_limit() const { + return hard_vocab_limit_; } inline bool TrainerSpec::hard_vocab_limit() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.hard_vocab_limit) - return hard_vocab_limit_; + return _internal_hard_vocab_limit(); } -inline void TrainerSpec::set_hard_vocab_limit(bool value) { - set_has_hard_vocab_limit(); +inline void TrainerSpec::_internal_set_hard_vocab_limit(bool value) { + _has_bits_[1] |= 0x00000001u; hard_vocab_limit_ = value; +} +inline void TrainerSpec::set_hard_vocab_limit(bool value) { + _internal_set_hard_vocab_limit(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.hard_vocab_limit) } // optional bool use_all_vocab = 34 [default = false]; -inline bool TrainerSpec::has_use_all_vocab() const { - return (_has_bits_[0] & 0x00008000u) != 0; +inline bool TrainerSpec::_internal_has_use_all_vocab() const { + bool value = (_has_bits_[0] & 0x00008000u) != 0; + return value; } -inline void TrainerSpec::set_has_use_all_vocab() { - _has_bits_[0] |= 0x00008000u; -} -inline void TrainerSpec::clear_has_use_all_vocab() { - _has_bits_[0] &= ~0x00008000u; +inline bool TrainerSpec::has_use_all_vocab() const { + return _internal_has_use_all_vocab(); } inline void TrainerSpec::clear_use_all_vocab() { use_all_vocab_ = false; - clear_has_use_all_vocab(); + _has_bits_[0] &= ~0x00008000u; +} +inline bool TrainerSpec::_internal_use_all_vocab() const { + return use_all_vocab_; } inline bool TrainerSpec::use_all_vocab() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.use_all_vocab) - return use_all_vocab_; + return _internal_use_all_vocab(); } -inline void TrainerSpec::set_use_all_vocab(bool value) { - set_has_use_all_vocab(); +inline void TrainerSpec::_internal_set_use_all_vocab(bool value) { + _has_bits_[0] |= 0x00008000u; use_all_vocab_ = value; +} +inline void TrainerSpec::set_use_all_vocab(bool value) { + _internal_set_use_all_vocab(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.use_all_vocab) } // optional int32 unk_id = 40 [default = 0]; -inline bool TrainerSpec::has_unk_id() const { - return (_has_bits_[0] & 0x00010000u) != 0; -} -inline void TrainerSpec::set_has_unk_id() { - _has_bits_[0] |= 0x00010000u; +inline bool TrainerSpec::_internal_has_unk_id() const { + bool value = (_has_bits_[0] & 0x00010000u) != 0; + return value; } -inline void TrainerSpec::clear_has_unk_id() { - _has_bits_[0] &= ~0x00010000u; +inline bool TrainerSpec::has_unk_id() const { + return _internal_has_unk_id(); } inline void TrainerSpec::clear_unk_id() { unk_id_ = 0; - clear_has_unk_id(); + _has_bits_[0] &= ~0x00010000u; } -inline ::google::protobuf::int32 TrainerSpec::unk_id() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.unk_id) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_unk_id() const { return unk_id_; } -inline void TrainerSpec::set_unk_id(::google::protobuf::int32 value) { - set_has_unk_id(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::unk_id() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.unk_id) + return _internal_unk_id(); +} +inline void TrainerSpec::_internal_set_unk_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[0] |= 0x00010000u; unk_id_ = value; +} +inline void TrainerSpec::set_unk_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_unk_id(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.unk_id) } // optional int32 bos_id = 41 [default = 1]; -inline bool TrainerSpec::has_bos_id() const { - return (_has_bits_[1] & 0x00000002u) != 0; +inline bool TrainerSpec::_internal_has_bos_id() const { + bool value = (_has_bits_[1] & 0x00000002u) != 0; + return value; } -inline void TrainerSpec::set_has_bos_id() { - _has_bits_[1] |= 0x00000002u; -} -inline void TrainerSpec::clear_has_bos_id() { - _has_bits_[1] &= ~0x00000002u; +inline bool TrainerSpec::has_bos_id() const { + return _internal_has_bos_id(); } inline void TrainerSpec::clear_bos_id() { bos_id_ = 1; - clear_has_bos_id(); + _has_bits_[1] &= ~0x00000002u; } -inline ::google::protobuf::int32 TrainerSpec::bos_id() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.bos_id) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_bos_id() const { return bos_id_; } -inline void TrainerSpec::set_bos_id(::google::protobuf::int32 value) { - set_has_bos_id(); - bos_id_ = value; - // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.bos_id) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::bos_id() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.bos_id) + return _internal_bos_id(); +} +inline void TrainerSpec::_internal_set_bos_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[1] |= 0x00000002u; + bos_id_ = value; +} +inline void TrainerSpec::set_bos_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_bos_id(value); + // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.bos_id) } // optional int32 eos_id = 42 [default = 2]; -inline bool TrainerSpec::has_eos_id() const { - return (_has_bits_[1] & 0x00000004u) != 0; +inline bool TrainerSpec::_internal_has_eos_id() const { + bool value = (_has_bits_[1] & 0x00000004u) != 0; + return value; } -inline void TrainerSpec::set_has_eos_id() { - _has_bits_[1] |= 0x00000004u; -} -inline void TrainerSpec::clear_has_eos_id() { - _has_bits_[1] &= ~0x00000004u; +inline bool TrainerSpec::has_eos_id() const { + return _internal_has_eos_id(); } inline void TrainerSpec::clear_eos_id() { eos_id_ = 2; - clear_has_eos_id(); + _has_bits_[1] &= ~0x00000004u; } -inline ::google::protobuf::int32 TrainerSpec::eos_id() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.eos_id) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_eos_id() const { return eos_id_; } -inline void TrainerSpec::set_eos_id(::google::protobuf::int32 value) { - set_has_eos_id(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::eos_id() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.eos_id) + return _internal_eos_id(); +} +inline void TrainerSpec::_internal_set_eos_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[1] |= 0x00000004u; eos_id_ = value; +} +inline void TrainerSpec::set_eos_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_eos_id(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.eos_id) } // optional int32 pad_id = 43 [default = -1]; -inline bool TrainerSpec::has_pad_id() const { - return (_has_bits_[1] & 0x00000008u) != 0; -} -inline void TrainerSpec::set_has_pad_id() { - _has_bits_[1] |= 0x00000008u; +inline bool TrainerSpec::_internal_has_pad_id() const { + bool value = (_has_bits_[1] & 0x00000008u) != 0; + return value; } -inline void TrainerSpec::clear_has_pad_id() { - _has_bits_[1] &= ~0x00000008u; +inline bool TrainerSpec::has_pad_id() const { + return _internal_has_pad_id(); } inline void TrainerSpec::clear_pad_id() { pad_id_ = -1; - clear_has_pad_id(); + _has_bits_[1] &= ~0x00000008u; } -inline ::google::protobuf::int32 TrainerSpec::pad_id() const { - // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.pad_id) +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::_internal_pad_id() const { return pad_id_; } -inline void TrainerSpec::set_pad_id(::google::protobuf::int32 value) { - set_has_pad_id(); +inline ::PROTOBUF_NAMESPACE_ID::int32 TrainerSpec::pad_id() const { + // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.pad_id) + return _internal_pad_id(); +} +inline void TrainerSpec::_internal_set_pad_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _has_bits_[1] |= 0x00000008u; pad_id_ = value; +} +inline void TrainerSpec::set_pad_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_pad_id(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.pad_id) } // optional string unk_piece = 45 [default = ""]; -inline bool TrainerSpec::has_unk_piece() const { - return (_has_bits_[0] & 0x00000010u) != 0; +inline bool TrainerSpec::_internal_has_unk_piece() const { + bool value = (_has_bits_[0] & 0x00000010u) != 0; + return value; } -inline void TrainerSpec::set_has_unk_piece() { - _has_bits_[0] |= 0x00000010u; -} -inline void TrainerSpec::clear_has_unk_piece() { - _has_bits_[0] &= ~0x00000010u; +inline bool TrainerSpec::has_unk_piece() const { + return _internal_has_unk_piece(); } inline void TrainerSpec::clear_unk_piece() { - unk_piece_.ClearToDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get()); - clear_has_unk_piece(); + unk_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), GetArena()); + _has_bits_[0] &= ~0x00000010u; } -inline const ::std::string& TrainerSpec::unk_piece() const { +inline const std::string& TrainerSpec::unk_piece() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.unk_piece) - return unk_piece_.GetNoArena(); + return _internal_unk_piece(); } -inline void TrainerSpec::set_unk_piece(const ::std::string& value) { - set_has_unk_piece(); - unk_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), value); +inline void TrainerSpec::set_unk_piece(const std::string& value) { + _internal_set_unk_piece(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.unk_piece) } -#if LANG_CXX11 -inline void TrainerSpec::set_unk_piece(::std::string&& value) { - set_has_unk_piece(); - unk_piece_.SetNoArena( - &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_unk_piece() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.unk_piece) + return _internal_mutable_unk_piece(); +} +inline const std::string& TrainerSpec::_internal_unk_piece() const { + return unk_piece_.Get(); +} +inline void TrainerSpec::_internal_set_unk_piece(const std::string& value) { + _has_bits_[0] |= 0x00000010u; + unk_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), value, GetArena()); +} +inline void TrainerSpec::set_unk_piece(std::string&& value) { + _has_bits_[0] |= 0x00000010u; + unk_piece_.SetLite( + &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.unk_piece) } -#endif inline void TrainerSpec::set_unk_piece(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_unk_piece(); - unk_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000010u; + unk_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.unk_piece) } -inline void TrainerSpec::set_unk_piece(const char* value, size_t size) { - set_has_unk_piece(); - unk_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_unk_piece(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000010u; + unk_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.unk_piece) } -inline ::std::string* TrainerSpec::mutable_unk_piece() { - set_has_unk_piece(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.unk_piece) - return unk_piece_.MutableNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get()); +inline std::string* TrainerSpec::_internal_mutable_unk_piece() { + _has_bits_[0] |= 0x00000010u; + return unk_piece_.Mutable(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), GetArena()); } -inline ::std::string* TrainerSpec::release_unk_piece() { +inline std::string* TrainerSpec::release_unk_piece() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.unk_piece) - if (!has_unk_piece()) { - return NULL; + if (!_internal_has_unk_piece()) { + return nullptr; } - clear_has_unk_piece(); - return unk_piece_.ReleaseNonDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get()); + _has_bits_[0] &= ~0x00000010u; + return unk_piece_.ReleaseNonDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), GetArena()); } -inline void TrainerSpec::set_allocated_unk_piece(::std::string* unk_piece) { - if (unk_piece != NULL) { - set_has_unk_piece(); +inline void TrainerSpec::set_allocated_unk_piece(std::string* unk_piece) { + if (unk_piece != nullptr) { + _has_bits_[0] |= 0x00000010u; } else { - clear_has_unk_piece(); + _has_bits_[0] &= ~0x00000010u; } - unk_piece_.SetAllocatedNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), unk_piece); + unk_piece_.SetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), unk_piece, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.unk_piece) } +inline std::string* TrainerSpec::unsafe_arena_release_unk_piece() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.unk_piece) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000010u; + return unk_piece_.UnsafeArenaRelease(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_unk_piece( + std::string* unk_piece) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (unk_piece != nullptr) { + _has_bits_[0] |= 0x00000010u; + } else { + _has_bits_[0] &= ~0x00000010u; + } + unk_piece_.UnsafeArenaSetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_.get(), + unk_piece, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.unk_piece) +} // optional string bos_piece = 46 [default = ""]; -inline bool TrainerSpec::has_bos_piece() const { - return (_has_bits_[0] & 0x00000020u) != 0; -} -inline void TrainerSpec::set_has_bos_piece() { - _has_bits_[0] |= 0x00000020u; +inline bool TrainerSpec::_internal_has_bos_piece() const { + bool value = (_has_bits_[0] & 0x00000020u) != 0; + return value; } -inline void TrainerSpec::clear_has_bos_piece() { - _has_bits_[0] &= ~0x00000020u; +inline bool TrainerSpec::has_bos_piece() const { + return _internal_has_bos_piece(); } inline void TrainerSpec::clear_bos_piece() { - bos_piece_.ClearToDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get()); - clear_has_bos_piece(); + bos_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), GetArena()); + _has_bits_[0] &= ~0x00000020u; } -inline const ::std::string& TrainerSpec::bos_piece() const { +inline const std::string& TrainerSpec::bos_piece() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.bos_piece) - return bos_piece_.GetNoArena(); + return _internal_bos_piece(); } -inline void TrainerSpec::set_bos_piece(const ::std::string& value) { - set_has_bos_piece(); - bos_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), value); +inline void TrainerSpec::set_bos_piece(const std::string& value) { + _internal_set_bos_piece(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.bos_piece) } -#if LANG_CXX11 -inline void TrainerSpec::set_bos_piece(::std::string&& value) { - set_has_bos_piece(); - bos_piece_.SetNoArena( - &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_bos_piece() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.bos_piece) + return _internal_mutable_bos_piece(); +} +inline const std::string& TrainerSpec::_internal_bos_piece() const { + return bos_piece_.Get(); +} +inline void TrainerSpec::_internal_set_bos_piece(const std::string& value) { + _has_bits_[0] |= 0x00000020u; + bos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), value, GetArena()); +} +inline void TrainerSpec::set_bos_piece(std::string&& value) { + _has_bits_[0] |= 0x00000020u; + bos_piece_.SetLite( + &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.bos_piece) } -#endif inline void TrainerSpec::set_bos_piece(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_bos_piece(); - bos_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000020u; + bos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.bos_piece) } -inline void TrainerSpec::set_bos_piece(const char* value, size_t size) { - set_has_bos_piece(); - bos_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_bos_piece(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000020u; + bos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.bos_piece) } -inline ::std::string* TrainerSpec::mutable_bos_piece() { - set_has_bos_piece(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.bos_piece) - return bos_piece_.MutableNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get()); +inline std::string* TrainerSpec::_internal_mutable_bos_piece() { + _has_bits_[0] |= 0x00000020u; + return bos_piece_.Mutable(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), GetArena()); } -inline ::std::string* TrainerSpec::release_bos_piece() { +inline std::string* TrainerSpec::release_bos_piece() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.bos_piece) - if (!has_bos_piece()) { - return NULL; + if (!_internal_has_bos_piece()) { + return nullptr; } - clear_has_bos_piece(); - return bos_piece_.ReleaseNonDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get()); + _has_bits_[0] &= ~0x00000020u; + return bos_piece_.ReleaseNonDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), GetArena()); } -inline void TrainerSpec::set_allocated_bos_piece(::std::string* bos_piece) { - if (bos_piece != NULL) { - set_has_bos_piece(); +inline void TrainerSpec::set_allocated_bos_piece(std::string* bos_piece) { + if (bos_piece != nullptr) { + _has_bits_[0] |= 0x00000020u; } else { - clear_has_bos_piece(); + _has_bits_[0] &= ~0x00000020u; } - bos_piece_.SetAllocatedNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), bos_piece); + bos_piece_.SetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), bos_piece, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.bos_piece) } +inline std::string* TrainerSpec::unsafe_arena_release_bos_piece() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.bos_piece) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000020u; + return bos_piece_.UnsafeArenaRelease(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_bos_piece( + std::string* bos_piece) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (bos_piece != nullptr) { + _has_bits_[0] |= 0x00000020u; + } else { + _has_bits_[0] &= ~0x00000020u; + } + bos_piece_.UnsafeArenaSetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_.get(), + bos_piece, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.bos_piece) +} // optional string eos_piece = 47 [default = ""]; -inline bool TrainerSpec::has_eos_piece() const { - return (_has_bits_[0] & 0x00000040u) != 0; -} -inline void TrainerSpec::set_has_eos_piece() { - _has_bits_[0] |= 0x00000040u; +inline bool TrainerSpec::_internal_has_eos_piece() const { + bool value = (_has_bits_[0] & 0x00000040u) != 0; + return value; } -inline void TrainerSpec::clear_has_eos_piece() { - _has_bits_[0] &= ~0x00000040u; +inline bool TrainerSpec::has_eos_piece() const { + return _internal_has_eos_piece(); } inline void TrainerSpec::clear_eos_piece() { - eos_piece_.ClearToDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get()); - clear_has_eos_piece(); + eos_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), GetArena()); + _has_bits_[0] &= ~0x00000040u; } -inline const ::std::string& TrainerSpec::eos_piece() const { +inline const std::string& TrainerSpec::eos_piece() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.eos_piece) - return eos_piece_.GetNoArena(); + return _internal_eos_piece(); } -inline void TrainerSpec::set_eos_piece(const ::std::string& value) { - set_has_eos_piece(); - eos_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), value); +inline void TrainerSpec::set_eos_piece(const std::string& value) { + _internal_set_eos_piece(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.eos_piece) } -#if LANG_CXX11 -inline void TrainerSpec::set_eos_piece(::std::string&& value) { - set_has_eos_piece(); - eos_piece_.SetNoArena( - &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_eos_piece() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.eos_piece) + return _internal_mutable_eos_piece(); +} +inline const std::string& TrainerSpec::_internal_eos_piece() const { + return eos_piece_.Get(); +} +inline void TrainerSpec::_internal_set_eos_piece(const std::string& value) { + _has_bits_[0] |= 0x00000040u; + eos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), value, GetArena()); +} +inline void TrainerSpec::set_eos_piece(std::string&& value) { + _has_bits_[0] |= 0x00000040u; + eos_piece_.SetLite( + &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.eos_piece) } -#endif inline void TrainerSpec::set_eos_piece(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_eos_piece(); - eos_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000040u; + eos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.eos_piece) } -inline void TrainerSpec::set_eos_piece(const char* value, size_t size) { - set_has_eos_piece(); - eos_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_eos_piece(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000040u; + eos_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.eos_piece) } -inline ::std::string* TrainerSpec::mutable_eos_piece() { - set_has_eos_piece(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.eos_piece) - return eos_piece_.MutableNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get()); +inline std::string* TrainerSpec::_internal_mutable_eos_piece() { + _has_bits_[0] |= 0x00000040u; + return eos_piece_.Mutable(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), GetArena()); } -inline ::std::string* TrainerSpec::release_eos_piece() { +inline std::string* TrainerSpec::release_eos_piece() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.eos_piece) - if (!has_eos_piece()) { - return NULL; + if (!_internal_has_eos_piece()) { + return nullptr; } - clear_has_eos_piece(); - return eos_piece_.ReleaseNonDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get()); + _has_bits_[0] &= ~0x00000040u; + return eos_piece_.ReleaseNonDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), GetArena()); } -inline void TrainerSpec::set_allocated_eos_piece(::std::string* eos_piece) { - if (eos_piece != NULL) { - set_has_eos_piece(); +inline void TrainerSpec::set_allocated_eos_piece(std::string* eos_piece) { + if (eos_piece != nullptr) { + _has_bits_[0] |= 0x00000040u; } else { - clear_has_eos_piece(); + _has_bits_[0] &= ~0x00000040u; } - eos_piece_.SetAllocatedNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), eos_piece); + eos_piece_.SetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), eos_piece, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.eos_piece) } +inline std::string* TrainerSpec::unsafe_arena_release_eos_piece() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.eos_piece) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000040u; + return eos_piece_.UnsafeArenaRelease(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_eos_piece( + std::string* eos_piece) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (eos_piece != nullptr) { + _has_bits_[0] |= 0x00000040u; + } else { + _has_bits_[0] &= ~0x00000040u; + } + eos_piece_.UnsafeArenaSetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_.get(), + eos_piece, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.eos_piece) +} // optional string pad_piece = 48 [default = ""]; -inline bool TrainerSpec::has_pad_piece() const { - return (_has_bits_[0] & 0x00000080u) != 0; -} -inline void TrainerSpec::set_has_pad_piece() { - _has_bits_[0] |= 0x00000080u; +inline bool TrainerSpec::_internal_has_pad_piece() const { + bool value = (_has_bits_[0] & 0x00000080u) != 0; + return value; } -inline void TrainerSpec::clear_has_pad_piece() { - _has_bits_[0] &= ~0x00000080u; +inline bool TrainerSpec::has_pad_piece() const { + return _internal_has_pad_piece(); } inline void TrainerSpec::clear_pad_piece() { - pad_piece_.ClearToDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get()); - clear_has_pad_piece(); + pad_piece_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), GetArena()); + _has_bits_[0] &= ~0x00000080u; } -inline const ::std::string& TrainerSpec::pad_piece() const { +inline const std::string& TrainerSpec::pad_piece() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.pad_piece) - return pad_piece_.GetNoArena(); + return _internal_pad_piece(); } -inline void TrainerSpec::set_pad_piece(const ::std::string& value) { - set_has_pad_piece(); - pad_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), value); +inline void TrainerSpec::set_pad_piece(const std::string& value) { + _internal_set_pad_piece(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.pad_piece) } -#if LANG_CXX11 -inline void TrainerSpec::set_pad_piece(::std::string&& value) { - set_has_pad_piece(); - pad_piece_.SetNoArena( - &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_pad_piece() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.pad_piece) + return _internal_mutable_pad_piece(); +} +inline const std::string& TrainerSpec::_internal_pad_piece() const { + return pad_piece_.Get(); +} +inline void TrainerSpec::_internal_set_pad_piece(const std::string& value) { + _has_bits_[0] |= 0x00000080u; + pad_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), value, GetArena()); +} +inline void TrainerSpec::set_pad_piece(std::string&& value) { + _has_bits_[0] |= 0x00000080u; + pad_piece_.SetLite( + &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.pad_piece) } -#endif inline void TrainerSpec::set_pad_piece(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_pad_piece(); - pad_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000080u; + pad_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.pad_piece) } -inline void TrainerSpec::set_pad_piece(const char* value, size_t size) { - set_has_pad_piece(); - pad_piece_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_pad_piece(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000080u; + pad_piece_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.pad_piece) } -inline ::std::string* TrainerSpec::mutable_pad_piece() { - set_has_pad_piece(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.pad_piece) - return pad_piece_.MutableNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get()); +inline std::string* TrainerSpec::_internal_mutable_pad_piece() { + _has_bits_[0] |= 0x00000080u; + return pad_piece_.Mutable(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), GetArena()); } -inline ::std::string* TrainerSpec::release_pad_piece() { +inline std::string* TrainerSpec::release_pad_piece() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.pad_piece) - if (!has_pad_piece()) { - return NULL; + if (!_internal_has_pad_piece()) { + return nullptr; } - clear_has_pad_piece(); - return pad_piece_.ReleaseNonDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get()); + _has_bits_[0] &= ~0x00000080u; + return pad_piece_.ReleaseNonDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), GetArena()); } -inline void TrainerSpec::set_allocated_pad_piece(::std::string* pad_piece) { - if (pad_piece != NULL) { - set_has_pad_piece(); +inline void TrainerSpec::set_allocated_pad_piece(std::string* pad_piece) { + if (pad_piece != nullptr) { + _has_bits_[0] |= 0x00000080u; } else { - clear_has_pad_piece(); + _has_bits_[0] &= ~0x00000080u; } - pad_piece_.SetAllocatedNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), pad_piece); + pad_piece_.SetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), pad_piece, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.pad_piece) } +inline std::string* TrainerSpec::unsafe_arena_release_pad_piece() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.pad_piece) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000080u; + return pad_piece_.UnsafeArenaRelease(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_pad_piece( + std::string* pad_piece) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (pad_piece != nullptr) { + _has_bits_[0] |= 0x00000080u; + } else { + _has_bits_[0] &= ~0x00000080u; + } + pad_piece_.UnsafeArenaSetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_.get(), + pad_piece, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.pad_piece) +} // optional string unk_surface = 44 [default = " \342\201\207 "]; -inline bool TrainerSpec::has_unk_surface() const { - return (_has_bits_[0] & 0x00000008u) != 0; +inline bool TrainerSpec::_internal_has_unk_surface() const { + bool value = (_has_bits_[0] & 0x00000008u) != 0; + return value; } -inline void TrainerSpec::set_has_unk_surface() { - _has_bits_[0] |= 0x00000008u; -} -inline void TrainerSpec::clear_has_unk_surface() { - _has_bits_[0] &= ~0x00000008u; +inline bool TrainerSpec::has_unk_surface() const { + return _internal_has_unk_surface(); } inline void TrainerSpec::clear_unk_surface() { - unk_surface_.ClearToDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get()); - clear_has_unk_surface(); + unk_surface_.ClearToDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), GetArena()); + _has_bits_[0] &= ~0x00000008u; } -inline const ::std::string& TrainerSpec::unk_surface() const { +inline const std::string& TrainerSpec::unk_surface() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.unk_surface) - return unk_surface_.GetNoArena(); + return _internal_unk_surface(); } -inline void TrainerSpec::set_unk_surface(const ::std::string& value) { - set_has_unk_surface(); - unk_surface_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), value); +inline void TrainerSpec::set_unk_surface(const std::string& value) { + _internal_set_unk_surface(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.unk_surface) } -#if LANG_CXX11 -inline void TrainerSpec::set_unk_surface(::std::string&& value) { - set_has_unk_surface(); - unk_surface_.SetNoArena( - &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), ::std::move(value)); +inline std::string* TrainerSpec::mutable_unk_surface() { + // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.unk_surface) + return _internal_mutable_unk_surface(); +} +inline const std::string& TrainerSpec::_internal_unk_surface() const { + return unk_surface_.Get(); +} +inline void TrainerSpec::_internal_set_unk_surface(const std::string& value) { + _has_bits_[0] |= 0x00000008u; + unk_surface_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), value, GetArena()); +} +inline void TrainerSpec::set_unk_surface(std::string&& value) { + _has_bits_[0] |= 0x00000008u; + unk_surface_.SetLite( + &::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.TrainerSpec.unk_surface) } -#endif inline void TrainerSpec::set_unk_surface(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_unk_surface(); - unk_surface_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000008u; + unk_surface_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.TrainerSpec.unk_surface) } -inline void TrainerSpec::set_unk_surface(const char* value, size_t size) { - set_has_unk_surface(); - unk_surface_.SetNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), - ::std::string(reinterpret_cast(value), size)); +inline void TrainerSpec::set_unk_surface(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000008u; + unk_surface_.SetLite(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.TrainerSpec.unk_surface) } -inline ::std::string* TrainerSpec::mutable_unk_surface() { - set_has_unk_surface(); - // @@protoc_insertion_point(field_mutable:sentencepiece.TrainerSpec.unk_surface) - return unk_surface_.MutableNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get()); +inline std::string* TrainerSpec::_internal_mutable_unk_surface() { + _has_bits_[0] |= 0x00000008u; + return unk_surface_.Mutable(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), GetArena()); } -inline ::std::string* TrainerSpec::release_unk_surface() { +inline std::string* TrainerSpec::release_unk_surface() { // @@protoc_insertion_point(field_release:sentencepiece.TrainerSpec.unk_surface) - if (!has_unk_surface()) { - return NULL; + if (!_internal_has_unk_surface()) { + return nullptr; } - clear_has_unk_surface(); - return unk_surface_.ReleaseNonDefaultNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get()); + _has_bits_[0] &= ~0x00000008u; + return unk_surface_.ReleaseNonDefault(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), GetArena()); } -inline void TrainerSpec::set_allocated_unk_surface(::std::string* unk_surface) { - if (unk_surface != NULL) { - set_has_unk_surface(); +inline void TrainerSpec::set_allocated_unk_surface(std::string* unk_surface) { + if (unk_surface != nullptr) { + _has_bits_[0] |= 0x00000008u; } else { - clear_has_unk_surface(); + _has_bits_[0] &= ~0x00000008u; } - unk_surface_.SetAllocatedNoArena(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), unk_surface); + unk_surface_.SetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), unk_surface, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.TrainerSpec.unk_surface) } +inline std::string* TrainerSpec::unsafe_arena_release_unk_surface() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.TrainerSpec.unk_surface) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000008u; + return unk_surface_.UnsafeArenaRelease(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), + GetArena()); +} +inline void TrainerSpec::unsafe_arena_set_allocated_unk_surface( + std::string* unk_surface) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (unk_surface != nullptr) { + _has_bits_[0] |= 0x00000008u; + } else { + _has_bits_[0] &= ~0x00000008u; + } + unk_surface_.UnsafeArenaSetAllocated(&::sentencepiece::TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_.get(), + unk_surface, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.TrainerSpec.unk_surface) +} // optional bool train_extremely_large_corpus = 49 [default = false]; -inline bool TrainerSpec::has_train_extremely_large_corpus() const { - return (_has_bits_[0] & 0x00020000u) != 0; -} -inline void TrainerSpec::set_has_train_extremely_large_corpus() { - _has_bits_[0] |= 0x00020000u; +inline bool TrainerSpec::_internal_has_train_extremely_large_corpus() const { + bool value = (_has_bits_[0] & 0x00020000u) != 0; + return value; } -inline void TrainerSpec::clear_has_train_extremely_large_corpus() { - _has_bits_[0] &= ~0x00020000u; +inline bool TrainerSpec::has_train_extremely_large_corpus() const { + return _internal_has_train_extremely_large_corpus(); } inline void TrainerSpec::clear_train_extremely_large_corpus() { train_extremely_large_corpus_ = false; - clear_has_train_extremely_large_corpus(); + _has_bits_[0] &= ~0x00020000u; +} +inline bool TrainerSpec::_internal_train_extremely_large_corpus() const { + return train_extremely_large_corpus_; } inline bool TrainerSpec::train_extremely_large_corpus() const { // @@protoc_insertion_point(field_get:sentencepiece.TrainerSpec.train_extremely_large_corpus) - return train_extremely_large_corpus_; + return _internal_train_extremely_large_corpus(); } -inline void TrainerSpec::set_train_extremely_large_corpus(bool value) { - set_has_train_extremely_large_corpus(); +inline void TrainerSpec::_internal_set_train_extremely_large_corpus(bool value) { + _has_bits_[0] |= 0x00020000u; train_extremely_large_corpus_ = value; +} +inline void TrainerSpec::set_train_extremely_large_corpus(bool value) { + _internal_set_train_extremely_large_corpus(value); // @@protoc_insertion_point(field_set:sentencepiece.TrainerSpec.train_extremely_large_corpus) } @@ -3041,419 +3931,569 @@ inline void TrainerSpec::set_train_extremely_large_corpus(bool value) { // NormalizerSpec // optional string name = 1; -inline bool NormalizerSpec::has_name() const { - return (_has_bits_[0] & 0x00000001u) != 0; +inline bool NormalizerSpec::_internal_has_name() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + return value; } -inline void NormalizerSpec::set_has_name() { - _has_bits_[0] |= 0x00000001u; -} -inline void NormalizerSpec::clear_has_name() { - _has_bits_[0] &= ~0x00000001u; +inline bool NormalizerSpec::has_name() const { + return _internal_has_name(); } inline void NormalizerSpec::clear_name() { - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_name(); + name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000001u; } -inline const ::std::string& NormalizerSpec::name() const { +inline const std::string& NormalizerSpec::name() const { // @@protoc_insertion_point(field_get:sentencepiece.NormalizerSpec.name) - return name_.GetNoArena(); + return _internal_name(); } -inline void NormalizerSpec::set_name(const ::std::string& value) { - set_has_name(); - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void NormalizerSpec::set_name(const std::string& value) { + _internal_set_name(value); // @@protoc_insertion_point(field_set:sentencepiece.NormalizerSpec.name) } -#if LANG_CXX11 -inline void NormalizerSpec::set_name(::std::string&& value) { - set_has_name(); - name_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* NormalizerSpec::mutable_name() { + // @@protoc_insertion_point(field_mutable:sentencepiece.NormalizerSpec.name) + return _internal_mutable_name(); +} +inline const std::string& NormalizerSpec::_internal_name() const { + return name_.Get(); +} +inline void NormalizerSpec::_internal_set_name(const std::string& value) { + _has_bits_[0] |= 0x00000001u; + name_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void NormalizerSpec::set_name(std::string&& value) { + _has_bits_[0] |= 0x00000001u; + name_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.NormalizerSpec.name) } -#endif inline void NormalizerSpec::set_name(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_name(); - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000001u; + name_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.NormalizerSpec.name) } -inline void NormalizerSpec::set_name(const char* value, size_t size) { - set_has_name(); - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void NormalizerSpec::set_name(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000001u; + name_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.NormalizerSpec.name) } -inline ::std::string* NormalizerSpec::mutable_name() { - set_has_name(); - // @@protoc_insertion_point(field_mutable:sentencepiece.NormalizerSpec.name) - return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* NormalizerSpec::_internal_mutable_name() { + _has_bits_[0] |= 0x00000001u; + return name_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* NormalizerSpec::release_name() { +inline std::string* NormalizerSpec::release_name() { // @@protoc_insertion_point(field_release:sentencepiece.NormalizerSpec.name) - if (!has_name()) { - return NULL; + if (!_internal_has_name()) { + return nullptr; } - clear_has_name(); - return name_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000001u; + return name_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void NormalizerSpec::set_allocated_name(::std::string* name) { - if (name != NULL) { - set_has_name(); +inline void NormalizerSpec::set_allocated_name(std::string* name) { + if (name != nullptr) { + _has_bits_[0] |= 0x00000001u; } else { - clear_has_name(); + _has_bits_[0] &= ~0x00000001u; } - name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name); + name_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), name, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.NormalizerSpec.name) } +inline std::string* NormalizerSpec::unsafe_arena_release_name() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.NormalizerSpec.name) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000001u; + return name_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void NormalizerSpec::unsafe_arena_set_allocated_name( + std::string* name) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (name != nullptr) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + name_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + name, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.NormalizerSpec.name) +} // optional bytes precompiled_charsmap = 2; -inline bool NormalizerSpec::has_precompiled_charsmap() const { - return (_has_bits_[0] & 0x00000002u) != 0; +inline bool NormalizerSpec::_internal_has_precompiled_charsmap() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + return value; } -inline void NormalizerSpec::set_has_precompiled_charsmap() { - _has_bits_[0] |= 0x00000002u; -} -inline void NormalizerSpec::clear_has_precompiled_charsmap() { - _has_bits_[0] &= ~0x00000002u; +inline bool NormalizerSpec::has_precompiled_charsmap() const { + return _internal_has_precompiled_charsmap(); } inline void NormalizerSpec::clear_precompiled_charsmap() { - precompiled_charsmap_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_precompiled_charsmap(); + precompiled_charsmap_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000002u; } -inline const ::std::string& NormalizerSpec::precompiled_charsmap() const { +inline const std::string& NormalizerSpec::precompiled_charsmap() const { // @@protoc_insertion_point(field_get:sentencepiece.NormalizerSpec.precompiled_charsmap) - return precompiled_charsmap_.GetNoArena(); + return _internal_precompiled_charsmap(); } -inline void NormalizerSpec::set_precompiled_charsmap(const ::std::string& value) { - set_has_precompiled_charsmap(); - precompiled_charsmap_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void NormalizerSpec::set_precompiled_charsmap(const std::string& value) { + _internal_set_precompiled_charsmap(value); // @@protoc_insertion_point(field_set:sentencepiece.NormalizerSpec.precompiled_charsmap) } -#if LANG_CXX11 -inline void NormalizerSpec::set_precompiled_charsmap(::std::string&& value) { - set_has_precompiled_charsmap(); - precompiled_charsmap_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* NormalizerSpec::mutable_precompiled_charsmap() { + // @@protoc_insertion_point(field_mutable:sentencepiece.NormalizerSpec.precompiled_charsmap) + return _internal_mutable_precompiled_charsmap(); +} +inline const std::string& NormalizerSpec::_internal_precompiled_charsmap() const { + return precompiled_charsmap_.Get(); +} +inline void NormalizerSpec::_internal_set_precompiled_charsmap(const std::string& value) { + _has_bits_[0] |= 0x00000002u; + precompiled_charsmap_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void NormalizerSpec::set_precompiled_charsmap(std::string&& value) { + _has_bits_[0] |= 0x00000002u; + precompiled_charsmap_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.NormalizerSpec.precompiled_charsmap) } -#endif inline void NormalizerSpec::set_precompiled_charsmap(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_precompiled_charsmap(); - precompiled_charsmap_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000002u; + precompiled_charsmap_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.NormalizerSpec.precompiled_charsmap) } -inline void NormalizerSpec::set_precompiled_charsmap(const void* value, size_t size) { - set_has_precompiled_charsmap(); - precompiled_charsmap_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void NormalizerSpec::set_precompiled_charsmap(const void* value, + size_t size) { + _has_bits_[0] |= 0x00000002u; + precompiled_charsmap_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.NormalizerSpec.precompiled_charsmap) } -inline ::std::string* NormalizerSpec::mutable_precompiled_charsmap() { - set_has_precompiled_charsmap(); - // @@protoc_insertion_point(field_mutable:sentencepiece.NormalizerSpec.precompiled_charsmap) - return precompiled_charsmap_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* NormalizerSpec::_internal_mutable_precompiled_charsmap() { + _has_bits_[0] |= 0x00000002u; + return precompiled_charsmap_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* NormalizerSpec::release_precompiled_charsmap() { +inline std::string* NormalizerSpec::release_precompiled_charsmap() { // @@protoc_insertion_point(field_release:sentencepiece.NormalizerSpec.precompiled_charsmap) - if (!has_precompiled_charsmap()) { - return NULL; + if (!_internal_has_precompiled_charsmap()) { + return nullptr; } - clear_has_precompiled_charsmap(); - return precompiled_charsmap_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000002u; + return precompiled_charsmap_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void NormalizerSpec::set_allocated_precompiled_charsmap(::std::string* precompiled_charsmap) { - if (precompiled_charsmap != NULL) { - set_has_precompiled_charsmap(); +inline void NormalizerSpec::set_allocated_precompiled_charsmap(std::string* precompiled_charsmap) { + if (precompiled_charsmap != nullptr) { + _has_bits_[0] |= 0x00000002u; } else { - clear_has_precompiled_charsmap(); + _has_bits_[0] &= ~0x00000002u; } - precompiled_charsmap_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), precompiled_charsmap); + precompiled_charsmap_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), precompiled_charsmap, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.NormalizerSpec.precompiled_charsmap) } +inline std::string* NormalizerSpec::unsafe_arena_release_precompiled_charsmap() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.NormalizerSpec.precompiled_charsmap) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000002u; + return precompiled_charsmap_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void NormalizerSpec::unsafe_arena_set_allocated_precompiled_charsmap( + std::string* precompiled_charsmap) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (precompiled_charsmap != nullptr) { + _has_bits_[0] |= 0x00000002u; + } else { + _has_bits_[0] &= ~0x00000002u; + } + precompiled_charsmap_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + precompiled_charsmap, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.NormalizerSpec.precompiled_charsmap) +} // optional bool add_dummy_prefix = 3 [default = true]; -inline bool NormalizerSpec::has_add_dummy_prefix() const { - return (_has_bits_[0] & 0x00000008u) != 0; +inline bool NormalizerSpec::_internal_has_add_dummy_prefix() const { + bool value = (_has_bits_[0] & 0x00000008u) != 0; + return value; } -inline void NormalizerSpec::set_has_add_dummy_prefix() { - _has_bits_[0] |= 0x00000008u; -} -inline void NormalizerSpec::clear_has_add_dummy_prefix() { - _has_bits_[0] &= ~0x00000008u; +inline bool NormalizerSpec::has_add_dummy_prefix() const { + return _internal_has_add_dummy_prefix(); } inline void NormalizerSpec::clear_add_dummy_prefix() { add_dummy_prefix_ = true; - clear_has_add_dummy_prefix(); + _has_bits_[0] &= ~0x00000008u; +} +inline bool NormalizerSpec::_internal_add_dummy_prefix() const { + return add_dummy_prefix_; } inline bool NormalizerSpec::add_dummy_prefix() const { // @@protoc_insertion_point(field_get:sentencepiece.NormalizerSpec.add_dummy_prefix) - return add_dummy_prefix_; + return _internal_add_dummy_prefix(); } -inline void NormalizerSpec::set_add_dummy_prefix(bool value) { - set_has_add_dummy_prefix(); +inline void NormalizerSpec::_internal_set_add_dummy_prefix(bool value) { + _has_bits_[0] |= 0x00000008u; add_dummy_prefix_ = value; +} +inline void NormalizerSpec::set_add_dummy_prefix(bool value) { + _internal_set_add_dummy_prefix(value); // @@protoc_insertion_point(field_set:sentencepiece.NormalizerSpec.add_dummy_prefix) } // optional bool remove_extra_whitespaces = 4 [default = true]; -inline bool NormalizerSpec::has_remove_extra_whitespaces() const { - return (_has_bits_[0] & 0x00000010u) != 0; -} -inline void NormalizerSpec::set_has_remove_extra_whitespaces() { - _has_bits_[0] |= 0x00000010u; +inline bool NormalizerSpec::_internal_has_remove_extra_whitespaces() const { + bool value = (_has_bits_[0] & 0x00000010u) != 0; + return value; } -inline void NormalizerSpec::clear_has_remove_extra_whitespaces() { - _has_bits_[0] &= ~0x00000010u; +inline bool NormalizerSpec::has_remove_extra_whitespaces() const { + return _internal_has_remove_extra_whitespaces(); } inline void NormalizerSpec::clear_remove_extra_whitespaces() { remove_extra_whitespaces_ = true; - clear_has_remove_extra_whitespaces(); + _has_bits_[0] &= ~0x00000010u; +} +inline bool NormalizerSpec::_internal_remove_extra_whitespaces() const { + return remove_extra_whitespaces_; } inline bool NormalizerSpec::remove_extra_whitespaces() const { // @@protoc_insertion_point(field_get:sentencepiece.NormalizerSpec.remove_extra_whitespaces) - return remove_extra_whitespaces_; + return _internal_remove_extra_whitespaces(); } -inline void NormalizerSpec::set_remove_extra_whitespaces(bool value) { - set_has_remove_extra_whitespaces(); +inline void NormalizerSpec::_internal_set_remove_extra_whitespaces(bool value) { + _has_bits_[0] |= 0x00000010u; remove_extra_whitespaces_ = value; +} +inline void NormalizerSpec::set_remove_extra_whitespaces(bool value) { + _internal_set_remove_extra_whitespaces(value); // @@protoc_insertion_point(field_set:sentencepiece.NormalizerSpec.remove_extra_whitespaces) } // optional bool escape_whitespaces = 5 [default = true]; -inline bool NormalizerSpec::has_escape_whitespaces() const { - return (_has_bits_[0] & 0x00000020u) != 0; -} -inline void NormalizerSpec::set_has_escape_whitespaces() { - _has_bits_[0] |= 0x00000020u; +inline bool NormalizerSpec::_internal_has_escape_whitespaces() const { + bool value = (_has_bits_[0] & 0x00000020u) != 0; + return value; } -inline void NormalizerSpec::clear_has_escape_whitespaces() { - _has_bits_[0] &= ~0x00000020u; +inline bool NormalizerSpec::has_escape_whitespaces() const { + return _internal_has_escape_whitespaces(); } inline void NormalizerSpec::clear_escape_whitespaces() { escape_whitespaces_ = true; - clear_has_escape_whitespaces(); + _has_bits_[0] &= ~0x00000020u; +} +inline bool NormalizerSpec::_internal_escape_whitespaces() const { + return escape_whitespaces_; } inline bool NormalizerSpec::escape_whitespaces() const { // @@protoc_insertion_point(field_get:sentencepiece.NormalizerSpec.escape_whitespaces) - return escape_whitespaces_; + return _internal_escape_whitespaces(); } -inline void NormalizerSpec::set_escape_whitespaces(bool value) { - set_has_escape_whitespaces(); +inline void NormalizerSpec::_internal_set_escape_whitespaces(bool value) { + _has_bits_[0] |= 0x00000020u; escape_whitespaces_ = value; +} +inline void NormalizerSpec::set_escape_whitespaces(bool value) { + _internal_set_escape_whitespaces(value); // @@protoc_insertion_point(field_set:sentencepiece.NormalizerSpec.escape_whitespaces) } // optional string normalization_rule_tsv = 6; -inline bool NormalizerSpec::has_normalization_rule_tsv() const { - return (_has_bits_[0] & 0x00000004u) != 0; +inline bool NormalizerSpec::_internal_has_normalization_rule_tsv() const { + bool value = (_has_bits_[0] & 0x00000004u) != 0; + return value; } -inline void NormalizerSpec::set_has_normalization_rule_tsv() { - _has_bits_[0] |= 0x00000004u; -} -inline void NormalizerSpec::clear_has_normalization_rule_tsv() { - _has_bits_[0] &= ~0x00000004u; +inline bool NormalizerSpec::has_normalization_rule_tsv() const { + return _internal_has_normalization_rule_tsv(); } inline void NormalizerSpec::clear_normalization_rule_tsv() { - normalization_rule_tsv_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_normalization_rule_tsv(); + normalization_rule_tsv_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000004u; } -inline const ::std::string& NormalizerSpec::normalization_rule_tsv() const { +inline const std::string& NormalizerSpec::normalization_rule_tsv() const { // @@protoc_insertion_point(field_get:sentencepiece.NormalizerSpec.normalization_rule_tsv) - return normalization_rule_tsv_.GetNoArena(); + return _internal_normalization_rule_tsv(); } -inline void NormalizerSpec::set_normalization_rule_tsv(const ::std::string& value) { - set_has_normalization_rule_tsv(); - normalization_rule_tsv_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void NormalizerSpec::set_normalization_rule_tsv(const std::string& value) { + _internal_set_normalization_rule_tsv(value); // @@protoc_insertion_point(field_set:sentencepiece.NormalizerSpec.normalization_rule_tsv) } -#if LANG_CXX11 -inline void NormalizerSpec::set_normalization_rule_tsv(::std::string&& value) { - set_has_normalization_rule_tsv(); - normalization_rule_tsv_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* NormalizerSpec::mutable_normalization_rule_tsv() { + // @@protoc_insertion_point(field_mutable:sentencepiece.NormalizerSpec.normalization_rule_tsv) + return _internal_mutable_normalization_rule_tsv(); +} +inline const std::string& NormalizerSpec::_internal_normalization_rule_tsv() const { + return normalization_rule_tsv_.Get(); +} +inline void NormalizerSpec::_internal_set_normalization_rule_tsv(const std::string& value) { + _has_bits_[0] |= 0x00000004u; + normalization_rule_tsv_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void NormalizerSpec::set_normalization_rule_tsv(std::string&& value) { + _has_bits_[0] |= 0x00000004u; + normalization_rule_tsv_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.NormalizerSpec.normalization_rule_tsv) } -#endif inline void NormalizerSpec::set_normalization_rule_tsv(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_normalization_rule_tsv(); - normalization_rule_tsv_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000004u; + normalization_rule_tsv_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.NormalizerSpec.normalization_rule_tsv) } -inline void NormalizerSpec::set_normalization_rule_tsv(const char* value, size_t size) { - set_has_normalization_rule_tsv(); - normalization_rule_tsv_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void NormalizerSpec::set_normalization_rule_tsv(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000004u; + normalization_rule_tsv_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.NormalizerSpec.normalization_rule_tsv) } -inline ::std::string* NormalizerSpec::mutable_normalization_rule_tsv() { - set_has_normalization_rule_tsv(); - // @@protoc_insertion_point(field_mutable:sentencepiece.NormalizerSpec.normalization_rule_tsv) - return normalization_rule_tsv_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* NormalizerSpec::_internal_mutable_normalization_rule_tsv() { + _has_bits_[0] |= 0x00000004u; + return normalization_rule_tsv_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* NormalizerSpec::release_normalization_rule_tsv() { +inline std::string* NormalizerSpec::release_normalization_rule_tsv() { // @@protoc_insertion_point(field_release:sentencepiece.NormalizerSpec.normalization_rule_tsv) - if (!has_normalization_rule_tsv()) { - return NULL; + if (!_internal_has_normalization_rule_tsv()) { + return nullptr; } - clear_has_normalization_rule_tsv(); - return normalization_rule_tsv_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000004u; + return normalization_rule_tsv_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void NormalizerSpec::set_allocated_normalization_rule_tsv(::std::string* normalization_rule_tsv) { - if (normalization_rule_tsv != NULL) { - set_has_normalization_rule_tsv(); +inline void NormalizerSpec::set_allocated_normalization_rule_tsv(std::string* normalization_rule_tsv) { + if (normalization_rule_tsv != nullptr) { + _has_bits_[0] |= 0x00000004u; } else { - clear_has_normalization_rule_tsv(); + _has_bits_[0] &= ~0x00000004u; } - normalization_rule_tsv_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), normalization_rule_tsv); + normalization_rule_tsv_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), normalization_rule_tsv, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.NormalizerSpec.normalization_rule_tsv) } +inline std::string* NormalizerSpec::unsafe_arena_release_normalization_rule_tsv() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.NormalizerSpec.normalization_rule_tsv) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000004u; + return normalization_rule_tsv_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void NormalizerSpec::unsafe_arena_set_allocated_normalization_rule_tsv( + std::string* normalization_rule_tsv) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (normalization_rule_tsv != nullptr) { + _has_bits_[0] |= 0x00000004u; + } else { + _has_bits_[0] &= ~0x00000004u; + } + normalization_rule_tsv_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + normalization_rule_tsv, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.NormalizerSpec.normalization_rule_tsv) +} // ------------------------------------------------------------------- // SelfTestData_Sample // optional string input = 1; -inline bool SelfTestData_Sample::has_input() const { - return (_has_bits_[0] & 0x00000001u) != 0; +inline bool SelfTestData_Sample::_internal_has_input() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + return value; } -inline void SelfTestData_Sample::set_has_input() { - _has_bits_[0] |= 0x00000001u; -} -inline void SelfTestData_Sample::clear_has_input() { - _has_bits_[0] &= ~0x00000001u; +inline bool SelfTestData_Sample::has_input() const { + return _internal_has_input(); } inline void SelfTestData_Sample::clear_input() { - input_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_input(); + input_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000001u; } -inline const ::std::string& SelfTestData_Sample::input() const { +inline const std::string& SelfTestData_Sample::input() const { // @@protoc_insertion_point(field_get:sentencepiece.SelfTestData.Sample.input) - return input_.GetNoArena(); + return _internal_input(); } -inline void SelfTestData_Sample::set_input(const ::std::string& value) { - set_has_input(); - input_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void SelfTestData_Sample::set_input(const std::string& value) { + _internal_set_input(value); // @@protoc_insertion_point(field_set:sentencepiece.SelfTestData.Sample.input) } -#if LANG_CXX11 -inline void SelfTestData_Sample::set_input(::std::string&& value) { - set_has_input(); - input_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* SelfTestData_Sample::mutable_input() { + // @@protoc_insertion_point(field_mutable:sentencepiece.SelfTestData.Sample.input) + return _internal_mutable_input(); +} +inline const std::string& SelfTestData_Sample::_internal_input() const { + return input_.Get(); +} +inline void SelfTestData_Sample::_internal_set_input(const std::string& value) { + _has_bits_[0] |= 0x00000001u; + input_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void SelfTestData_Sample::set_input(std::string&& value) { + _has_bits_[0] |= 0x00000001u; + input_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.SelfTestData.Sample.input) } -#endif inline void SelfTestData_Sample::set_input(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_input(); - input_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000001u; + input_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.SelfTestData.Sample.input) } -inline void SelfTestData_Sample::set_input(const char* value, size_t size) { - set_has_input(); - input_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void SelfTestData_Sample::set_input(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000001u; + input_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.SelfTestData.Sample.input) } -inline ::std::string* SelfTestData_Sample::mutable_input() { - set_has_input(); - // @@protoc_insertion_point(field_mutable:sentencepiece.SelfTestData.Sample.input) - return input_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* SelfTestData_Sample::_internal_mutable_input() { + _has_bits_[0] |= 0x00000001u; + return input_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* SelfTestData_Sample::release_input() { +inline std::string* SelfTestData_Sample::release_input() { // @@protoc_insertion_point(field_release:sentencepiece.SelfTestData.Sample.input) - if (!has_input()) { - return NULL; + if (!_internal_has_input()) { + return nullptr; } - clear_has_input(); - return input_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000001u; + return input_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void SelfTestData_Sample::set_allocated_input(::std::string* input) { - if (input != NULL) { - set_has_input(); +inline void SelfTestData_Sample::set_allocated_input(std::string* input) { + if (input != nullptr) { + _has_bits_[0] |= 0x00000001u; } else { - clear_has_input(); + _has_bits_[0] &= ~0x00000001u; } - input_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), input); + input_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), input, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.SelfTestData.Sample.input) } +inline std::string* SelfTestData_Sample::unsafe_arena_release_input() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.SelfTestData.Sample.input) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000001u; + return input_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void SelfTestData_Sample::unsafe_arena_set_allocated_input( + std::string* input) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (input != nullptr) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + input_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + input, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.SelfTestData.Sample.input) +} // optional string expected = 2; -inline bool SelfTestData_Sample::has_expected() const { - return (_has_bits_[0] & 0x00000002u) != 0; +inline bool SelfTestData_Sample::_internal_has_expected() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + return value; } -inline void SelfTestData_Sample::set_has_expected() { - _has_bits_[0] |= 0x00000002u; -} -inline void SelfTestData_Sample::clear_has_expected() { - _has_bits_[0] &= ~0x00000002u; +inline bool SelfTestData_Sample::has_expected() const { + return _internal_has_expected(); } inline void SelfTestData_Sample::clear_expected() { - expected_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_expected(); + expected_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000002u; } -inline const ::std::string& SelfTestData_Sample::expected() const { +inline const std::string& SelfTestData_Sample::expected() const { // @@protoc_insertion_point(field_get:sentencepiece.SelfTestData.Sample.expected) - return expected_.GetNoArena(); + return _internal_expected(); } -inline void SelfTestData_Sample::set_expected(const ::std::string& value) { - set_has_expected(); - expected_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void SelfTestData_Sample::set_expected(const std::string& value) { + _internal_set_expected(value); // @@protoc_insertion_point(field_set:sentencepiece.SelfTestData.Sample.expected) } -#if LANG_CXX11 -inline void SelfTestData_Sample::set_expected(::std::string&& value) { - set_has_expected(); - expected_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* SelfTestData_Sample::mutable_expected() { + // @@protoc_insertion_point(field_mutable:sentencepiece.SelfTestData.Sample.expected) + return _internal_mutable_expected(); +} +inline const std::string& SelfTestData_Sample::_internal_expected() const { + return expected_.Get(); +} +inline void SelfTestData_Sample::_internal_set_expected(const std::string& value) { + _has_bits_[0] |= 0x00000002u; + expected_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void SelfTestData_Sample::set_expected(std::string&& value) { + _has_bits_[0] |= 0x00000002u; + expected_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.SelfTestData.Sample.expected) } -#endif inline void SelfTestData_Sample::set_expected(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_expected(); - expected_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000002u; + expected_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.SelfTestData.Sample.expected) } -inline void SelfTestData_Sample::set_expected(const char* value, size_t size) { - set_has_expected(); - expected_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void SelfTestData_Sample::set_expected(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000002u; + expected_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.SelfTestData.Sample.expected) } -inline ::std::string* SelfTestData_Sample::mutable_expected() { - set_has_expected(); - // @@protoc_insertion_point(field_mutable:sentencepiece.SelfTestData.Sample.expected) - return expected_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* SelfTestData_Sample::_internal_mutable_expected() { + _has_bits_[0] |= 0x00000002u; + return expected_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* SelfTestData_Sample::release_expected() { +inline std::string* SelfTestData_Sample::release_expected() { // @@protoc_insertion_point(field_release:sentencepiece.SelfTestData.Sample.expected) - if (!has_expected()) { - return NULL; + if (!_internal_has_expected()) { + return nullptr; } - clear_has_expected(); - return expected_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000002u; + return expected_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void SelfTestData_Sample::set_allocated_expected(::std::string* expected) { - if (expected != NULL) { - set_has_expected(); +inline void SelfTestData_Sample::set_allocated_expected(std::string* expected) { + if (expected != nullptr) { + _has_bits_[0] |= 0x00000002u; } else { - clear_has_expected(); + _has_bits_[0] &= ~0x00000002u; } - expected_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), expected); + expected_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), expected, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.SelfTestData.Sample.expected) } +inline std::string* SelfTestData_Sample::unsafe_arena_release_expected() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.SelfTestData.Sample.expected) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000002u; + return expected_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void SelfTestData_Sample::unsafe_arena_set_allocated_expected( + std::string* expected) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (expected != nullptr) { + _has_bits_[0] |= 0x00000002u; + } else { + _has_bits_[0] &= ~0x00000002u; + } + expected_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + expected, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.SelfTestData.Sample.expected) +} // ------------------------------------------------------------------- // SelfTestData // repeated .sentencepiece.SelfTestData.Sample samples = 1; -inline int SelfTestData::samples_size() const { +inline int SelfTestData::_internal_samples_size() const { return samples_.size(); } +inline int SelfTestData::samples_size() const { + return _internal_samples_size(); +} inline void SelfTestData::clear_samples() { samples_.Clear(); } @@ -3461,20 +4501,26 @@ inline ::sentencepiece::SelfTestData_Sample* SelfTestData::mutable_samples(int i // @@protoc_insertion_point(field_mutable:sentencepiece.SelfTestData.samples) return samples_.Mutable(index); } -inline ::google::protobuf::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >* SelfTestData::mutable_samples() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.SelfTestData.samples) return &samples_; } +inline const ::sentencepiece::SelfTestData_Sample& SelfTestData::_internal_samples(int index) const { + return samples_.Get(index); +} inline const ::sentencepiece::SelfTestData_Sample& SelfTestData::samples(int index) const { // @@protoc_insertion_point(field_get:sentencepiece.SelfTestData.samples) - return samples_.Get(index); + return _internal_samples(index); +} +inline ::sentencepiece::SelfTestData_Sample* SelfTestData::_internal_add_samples() { + return samples_.Add(); } inline ::sentencepiece::SelfTestData_Sample* SelfTestData::add_samples() { // @@protoc_insertion_point(field_add:sentencepiece.SelfTestData.samples) - return samples_.Add(); + return _internal_add_samples(); } -inline const ::google::protobuf::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::SelfTestData_Sample >& SelfTestData::samples() const { // @@protoc_insertion_point(field_list:sentencepiece.SelfTestData.samples) return samples_; @@ -3485,117 +4531,152 @@ SelfTestData::samples() const { // ModelProto_SentencePiece // optional string piece = 1; -inline bool ModelProto_SentencePiece::has_piece() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void ModelProto_SentencePiece::set_has_piece() { - _has_bits_[0] |= 0x00000001u; +inline bool ModelProto_SentencePiece::_internal_has_piece() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + return value; } -inline void ModelProto_SentencePiece::clear_has_piece() { - _has_bits_[0] &= ~0x00000001u; +inline bool ModelProto_SentencePiece::has_piece() const { + return _internal_has_piece(); } inline void ModelProto_SentencePiece::clear_piece() { - piece_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - clear_has_piece(); + piece_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); + _has_bits_[0] &= ~0x00000001u; } -inline const ::std::string& ModelProto_SentencePiece::piece() const { +inline const std::string& ModelProto_SentencePiece::piece() const { // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.SentencePiece.piece) - return piece_.GetNoArena(); + return _internal_piece(); } -inline void ModelProto_SentencePiece::set_piece(const ::std::string& value) { - set_has_piece(); - piece_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); +inline void ModelProto_SentencePiece::set_piece(const std::string& value) { + _internal_set_piece(value); // @@protoc_insertion_point(field_set:sentencepiece.ModelProto.SentencePiece.piece) } -#if LANG_CXX11 -inline void ModelProto_SentencePiece::set_piece(::std::string&& value) { - set_has_piece(); - piece_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); +inline std::string* ModelProto_SentencePiece::mutable_piece() { + // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.SentencePiece.piece) + return _internal_mutable_piece(); +} +inline const std::string& ModelProto_SentencePiece::_internal_piece() const { + return piece_.Get(); +} +inline void ModelProto_SentencePiece::_internal_set_piece(const std::string& value) { + _has_bits_[0] |= 0x00000001u; + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); +} +inline void ModelProto_SentencePiece::set_piece(std::string&& value) { + _has_bits_[0] |= 0x00000001u; + piece_.SetLite( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); // @@protoc_insertion_point(field_set_rvalue:sentencepiece.ModelProto.SentencePiece.piece) } -#endif inline void ModelProto_SentencePiece::set_piece(const char* value) { - GOOGLE_DCHECK(value != NULL); - set_has_piece(); - piece_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + GOOGLE_DCHECK(value != nullptr); + _has_bits_[0] |= 0x00000001u; + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArena()); // @@protoc_insertion_point(field_set_char:sentencepiece.ModelProto.SentencePiece.piece) } -inline void ModelProto_SentencePiece::set_piece(const char* value, size_t size) { - set_has_piece(); - piece_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); +inline void ModelProto_SentencePiece::set_piece(const char* value, + size_t size) { + _has_bits_[0] |= 0x00000001u; + piece_.SetLite(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArena()); // @@protoc_insertion_point(field_set_pointer:sentencepiece.ModelProto.SentencePiece.piece) } -inline ::std::string* ModelProto_SentencePiece::mutable_piece() { - set_has_piece(); - // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.SentencePiece.piece) - return piece_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +inline std::string* ModelProto_SentencePiece::_internal_mutable_piece() { + _has_bits_[0] |= 0x00000001u; + return piece_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline ::std::string* ModelProto_SentencePiece::release_piece() { +inline std::string* ModelProto_SentencePiece::release_piece() { // @@protoc_insertion_point(field_release:sentencepiece.ModelProto.SentencePiece.piece) - if (!has_piece()) { - return NULL; + if (!_internal_has_piece()) { + return nullptr; } - clear_has_piece(); - return piece_.ReleaseNonDefaultNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + _has_bits_[0] &= ~0x00000001u; + return piece_.ReleaseNonDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); } -inline void ModelProto_SentencePiece::set_allocated_piece(::std::string* piece) { - if (piece != NULL) { - set_has_piece(); +inline void ModelProto_SentencePiece::set_allocated_piece(std::string* piece) { + if (piece != nullptr) { + _has_bits_[0] |= 0x00000001u; } else { - clear_has_piece(); + _has_bits_[0] &= ~0x00000001u; } - piece_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), piece); + piece_.SetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), piece, + GetArena()); // @@protoc_insertion_point(field_set_allocated:sentencepiece.ModelProto.SentencePiece.piece) } +inline std::string* ModelProto_SentencePiece::unsafe_arena_release_piece() { + // @@protoc_insertion_point(field_unsafe_arena_release:sentencepiece.ModelProto.SentencePiece.piece) + GOOGLE_DCHECK(GetArena() != nullptr); + _has_bits_[0] &= ~0x00000001u; + return piece_.UnsafeArenaRelease(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArena()); +} +inline void ModelProto_SentencePiece::unsafe_arena_set_allocated_piece( + std::string* piece) { + GOOGLE_DCHECK(GetArena() != nullptr); + if (piece != nullptr) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + piece_.UnsafeArenaSetAllocated(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + piece, GetArena()); + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.ModelProto.SentencePiece.piece) +} // optional float score = 2; -inline bool ModelProto_SentencePiece::has_score() const { - return (_has_bits_[0] & 0x00000002u) != 0; +inline bool ModelProto_SentencePiece::_internal_has_score() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + return value; } -inline void ModelProto_SentencePiece::set_has_score() { - _has_bits_[0] |= 0x00000002u; -} -inline void ModelProto_SentencePiece::clear_has_score() { - _has_bits_[0] &= ~0x00000002u; +inline bool ModelProto_SentencePiece::has_score() const { + return _internal_has_score(); } inline void ModelProto_SentencePiece::clear_score() { score_ = 0; - clear_has_score(); + _has_bits_[0] &= ~0x00000002u; +} +inline float ModelProto_SentencePiece::_internal_score() const { + return score_; } inline float ModelProto_SentencePiece::score() const { // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.SentencePiece.score) - return score_; + return _internal_score(); } -inline void ModelProto_SentencePiece::set_score(float value) { - set_has_score(); +inline void ModelProto_SentencePiece::_internal_set_score(float value) { + _has_bits_[0] |= 0x00000002u; score_ = value; +} +inline void ModelProto_SentencePiece::set_score(float value) { + _internal_set_score(value); // @@protoc_insertion_point(field_set:sentencepiece.ModelProto.SentencePiece.score) } // optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default = NORMAL]; -inline bool ModelProto_SentencePiece::has_type() const { - return (_has_bits_[0] & 0x00000004u) != 0; -} -inline void ModelProto_SentencePiece::set_has_type() { - _has_bits_[0] |= 0x00000004u; +inline bool ModelProto_SentencePiece::_internal_has_type() const { + bool value = (_has_bits_[0] & 0x00000004u) != 0; + return value; } -inline void ModelProto_SentencePiece::clear_has_type() { - _has_bits_[0] &= ~0x00000004u; +inline bool ModelProto_SentencePiece::has_type() const { + return _internal_has_type(); } inline void ModelProto_SentencePiece::clear_type() { type_ = 1; - clear_has_type(); + _has_bits_[0] &= ~0x00000004u; +} +inline ::sentencepiece::ModelProto_SentencePiece_Type ModelProto_SentencePiece::_internal_type() const { + return static_cast< ::sentencepiece::ModelProto_SentencePiece_Type >(type_); } inline ::sentencepiece::ModelProto_SentencePiece_Type ModelProto_SentencePiece::type() const { // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.SentencePiece.type) - return static_cast< ::sentencepiece::ModelProto_SentencePiece_Type >(type_); + return _internal_type(); } -inline void ModelProto_SentencePiece::set_type(::sentencepiece::ModelProto_SentencePiece_Type value) { +inline void ModelProto_SentencePiece::_internal_set_type(::sentencepiece::ModelProto_SentencePiece_Type value) { assert(::sentencepiece::ModelProto_SentencePiece_Type_IsValid(value)); - set_has_type(); + _has_bits_[0] |= 0x00000004u; type_ = value; +} +inline void ModelProto_SentencePiece::set_type(::sentencepiece::ModelProto_SentencePiece_Type value) { + _internal_set_type(value); // @@protoc_insertion_point(field_set:sentencepiece.ModelProto.SentencePiece.type) } @@ -3604,9 +4685,12 @@ inline void ModelProto_SentencePiece::set_type(::sentencepiece::ModelProto_Sente // ModelProto // repeated .sentencepiece.ModelProto.SentencePiece pieces = 1; -inline int ModelProto::pieces_size() const { +inline int ModelProto::_internal_pieces_size() const { return pieces_.size(); } +inline int ModelProto::pieces_size() const { + return _internal_pieces_size(); +} inline void ModelProto::clear_pieces() { pieces_.Clear(); } @@ -3614,252 +4698,350 @@ inline ::sentencepiece::ModelProto_SentencePiece* ModelProto::mutable_pieces(int // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.pieces) return pieces_.Mutable(index); } -inline ::google::protobuf::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >* +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >* ModelProto::mutable_pieces() { // @@protoc_insertion_point(field_mutable_list:sentencepiece.ModelProto.pieces) return &pieces_; } +inline const ::sentencepiece::ModelProto_SentencePiece& ModelProto::_internal_pieces(int index) const { + return pieces_.Get(index); +} inline const ::sentencepiece::ModelProto_SentencePiece& ModelProto::pieces(int index) const { // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.pieces) - return pieces_.Get(index); + return _internal_pieces(index); +} +inline ::sentencepiece::ModelProto_SentencePiece* ModelProto::_internal_add_pieces() { + return pieces_.Add(); } inline ::sentencepiece::ModelProto_SentencePiece* ModelProto::add_pieces() { // @@protoc_insertion_point(field_add:sentencepiece.ModelProto.pieces) - return pieces_.Add(); + return _internal_add_pieces(); } -inline const ::google::protobuf::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >& +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::sentencepiece::ModelProto_SentencePiece >& ModelProto::pieces() const { // @@protoc_insertion_point(field_list:sentencepiece.ModelProto.pieces) return pieces_; } // optional .sentencepiece.TrainerSpec trainer_spec = 2; -inline bool ModelProto::has_trainer_spec() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void ModelProto::set_has_trainer_spec() { - _has_bits_[0] |= 0x00000001u; +inline bool ModelProto::_internal_has_trainer_spec() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + PROTOBUF_ASSUME(!value || trainer_spec_ != nullptr); + return value; } -inline void ModelProto::clear_has_trainer_spec() { - _has_bits_[0] &= ~0x00000001u; +inline bool ModelProto::has_trainer_spec() const { + return _internal_has_trainer_spec(); } inline void ModelProto::clear_trainer_spec() { - if (trainer_spec_ != NULL) trainer_spec_->Clear(); - clear_has_trainer_spec(); + if (trainer_spec_ != nullptr) trainer_spec_->Clear(); + _has_bits_[0] &= ~0x00000001u; } inline const ::sentencepiece::TrainerSpec& ModelProto::_internal_trainer_spec() const { - return *trainer_spec_; + const ::sentencepiece::TrainerSpec* p = trainer_spec_; + return p != nullptr ? *p : *reinterpret_cast( + &::sentencepiece::_TrainerSpec_default_instance_); } inline const ::sentencepiece::TrainerSpec& ModelProto::trainer_spec() const { - const ::sentencepiece::TrainerSpec* p = trainer_spec_; // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.trainer_spec) - return p != NULL ? *p : *reinterpret_cast( - &::sentencepiece::_TrainerSpec_default_instance_); + return _internal_trainer_spec(); +} +inline void ModelProto::unsafe_arena_set_allocated_trainer_spec( + ::sentencepiece::TrainerSpec* trainer_spec) { + if (GetArena() == nullptr) { + delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(trainer_spec_); + } + trainer_spec_ = trainer_spec; + if (trainer_spec) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.ModelProto.trainer_spec) } inline ::sentencepiece::TrainerSpec* ModelProto::release_trainer_spec() { + auto temp = unsafe_arena_release_trainer_spec(); + if (GetArena() != nullptr) { + temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); + } + return temp; +} +inline ::sentencepiece::TrainerSpec* ModelProto::unsafe_arena_release_trainer_spec() { // @@protoc_insertion_point(field_release:sentencepiece.ModelProto.trainer_spec) - clear_has_trainer_spec(); + _has_bits_[0] &= ~0x00000001u; ::sentencepiece::TrainerSpec* temp = trainer_spec_; - trainer_spec_ = NULL; + trainer_spec_ = nullptr; return temp; } -inline ::sentencepiece::TrainerSpec* ModelProto::mutable_trainer_spec() { - set_has_trainer_spec(); - if (trainer_spec_ == NULL) { - auto* p = CreateMaybeMessage<::sentencepiece::TrainerSpec>(GetArenaNoVirtual()); +inline ::sentencepiece::TrainerSpec* ModelProto::_internal_mutable_trainer_spec() { + _has_bits_[0] |= 0x00000001u; + if (trainer_spec_ == nullptr) { + auto* p = CreateMaybeMessage<::sentencepiece::TrainerSpec>(GetArena()); trainer_spec_ = p; } - // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.trainer_spec) return trainer_spec_; } +inline ::sentencepiece::TrainerSpec* ModelProto::mutable_trainer_spec() { + // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.trainer_spec) + return _internal_mutable_trainer_spec(); +} inline void ModelProto::set_allocated_trainer_spec(::sentencepiece::TrainerSpec* trainer_spec) { - ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == NULL) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArena(); + if (message_arena == nullptr) { delete trainer_spec_; } if (trainer_spec) { - ::google::protobuf::Arena* submessage_arena = NULL; + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = + ::PROTOBUF_NAMESPACE_ID::Arena::GetArena(trainer_spec); if (message_arena != submessage_arena) { - trainer_spec = ::google::protobuf::internal::GetOwnedMessage( + trainer_spec = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, trainer_spec, submessage_arena); } - set_has_trainer_spec(); + _has_bits_[0] |= 0x00000001u; } else { - clear_has_trainer_spec(); + _has_bits_[0] &= ~0x00000001u; } trainer_spec_ = trainer_spec; // @@protoc_insertion_point(field_set_allocated:sentencepiece.ModelProto.trainer_spec) } // optional .sentencepiece.NormalizerSpec normalizer_spec = 3; -inline bool ModelProto::has_normalizer_spec() const { - return (_has_bits_[0] & 0x00000002u) != 0; -} -inline void ModelProto::set_has_normalizer_spec() { - _has_bits_[0] |= 0x00000002u; +inline bool ModelProto::_internal_has_normalizer_spec() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + PROTOBUF_ASSUME(!value || normalizer_spec_ != nullptr); + return value; } -inline void ModelProto::clear_has_normalizer_spec() { - _has_bits_[0] &= ~0x00000002u; +inline bool ModelProto::has_normalizer_spec() const { + return _internal_has_normalizer_spec(); } inline void ModelProto::clear_normalizer_spec() { - if (normalizer_spec_ != NULL) normalizer_spec_->Clear(); - clear_has_normalizer_spec(); + if (normalizer_spec_ != nullptr) normalizer_spec_->Clear(); + _has_bits_[0] &= ~0x00000002u; } inline const ::sentencepiece::NormalizerSpec& ModelProto::_internal_normalizer_spec() const { - return *normalizer_spec_; + const ::sentencepiece::NormalizerSpec* p = normalizer_spec_; + return p != nullptr ? *p : *reinterpret_cast( + &::sentencepiece::_NormalizerSpec_default_instance_); } inline const ::sentencepiece::NormalizerSpec& ModelProto::normalizer_spec() const { - const ::sentencepiece::NormalizerSpec* p = normalizer_spec_; // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.normalizer_spec) - return p != NULL ? *p : *reinterpret_cast( - &::sentencepiece::_NormalizerSpec_default_instance_); + return _internal_normalizer_spec(); +} +inline void ModelProto::unsafe_arena_set_allocated_normalizer_spec( + ::sentencepiece::NormalizerSpec* normalizer_spec) { + if (GetArena() == nullptr) { + delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(normalizer_spec_); + } + normalizer_spec_ = normalizer_spec; + if (normalizer_spec) { + _has_bits_[0] |= 0x00000002u; + } else { + _has_bits_[0] &= ~0x00000002u; + } + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.ModelProto.normalizer_spec) } inline ::sentencepiece::NormalizerSpec* ModelProto::release_normalizer_spec() { + auto temp = unsafe_arena_release_normalizer_spec(); + if (GetArena() != nullptr) { + temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); + } + return temp; +} +inline ::sentencepiece::NormalizerSpec* ModelProto::unsafe_arena_release_normalizer_spec() { // @@protoc_insertion_point(field_release:sentencepiece.ModelProto.normalizer_spec) - clear_has_normalizer_spec(); + _has_bits_[0] &= ~0x00000002u; ::sentencepiece::NormalizerSpec* temp = normalizer_spec_; - normalizer_spec_ = NULL; + normalizer_spec_ = nullptr; return temp; } -inline ::sentencepiece::NormalizerSpec* ModelProto::mutable_normalizer_spec() { - set_has_normalizer_spec(); - if (normalizer_spec_ == NULL) { - auto* p = CreateMaybeMessage<::sentencepiece::NormalizerSpec>(GetArenaNoVirtual()); +inline ::sentencepiece::NormalizerSpec* ModelProto::_internal_mutable_normalizer_spec() { + _has_bits_[0] |= 0x00000002u; + if (normalizer_spec_ == nullptr) { + auto* p = CreateMaybeMessage<::sentencepiece::NormalizerSpec>(GetArena()); normalizer_spec_ = p; } - // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.normalizer_spec) return normalizer_spec_; } +inline ::sentencepiece::NormalizerSpec* ModelProto::mutable_normalizer_spec() { + // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.normalizer_spec) + return _internal_mutable_normalizer_spec(); +} inline void ModelProto::set_allocated_normalizer_spec(::sentencepiece::NormalizerSpec* normalizer_spec) { - ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == NULL) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArena(); + if (message_arena == nullptr) { delete normalizer_spec_; } if (normalizer_spec) { - ::google::protobuf::Arena* submessage_arena = NULL; + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = + ::PROTOBUF_NAMESPACE_ID::Arena::GetArena(normalizer_spec); if (message_arena != submessage_arena) { - normalizer_spec = ::google::protobuf::internal::GetOwnedMessage( + normalizer_spec = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, normalizer_spec, submessage_arena); } - set_has_normalizer_spec(); + _has_bits_[0] |= 0x00000002u; } else { - clear_has_normalizer_spec(); + _has_bits_[0] &= ~0x00000002u; } normalizer_spec_ = normalizer_spec; // @@protoc_insertion_point(field_set_allocated:sentencepiece.ModelProto.normalizer_spec) } // optional .sentencepiece.SelfTestData self_test_data = 4; -inline bool ModelProto::has_self_test_data() const { - return (_has_bits_[0] & 0x00000004u) != 0; +inline bool ModelProto::_internal_has_self_test_data() const { + bool value = (_has_bits_[0] & 0x00000004u) != 0; + PROTOBUF_ASSUME(!value || self_test_data_ != nullptr); + return value; } -inline void ModelProto::set_has_self_test_data() { - _has_bits_[0] |= 0x00000004u; -} -inline void ModelProto::clear_has_self_test_data() { - _has_bits_[0] &= ~0x00000004u; +inline bool ModelProto::has_self_test_data() const { + return _internal_has_self_test_data(); } inline void ModelProto::clear_self_test_data() { - if (self_test_data_ != NULL) self_test_data_->Clear(); - clear_has_self_test_data(); + if (self_test_data_ != nullptr) self_test_data_->Clear(); + _has_bits_[0] &= ~0x00000004u; } inline const ::sentencepiece::SelfTestData& ModelProto::_internal_self_test_data() const { - return *self_test_data_; + const ::sentencepiece::SelfTestData* p = self_test_data_; + return p != nullptr ? *p : *reinterpret_cast( + &::sentencepiece::_SelfTestData_default_instance_); } inline const ::sentencepiece::SelfTestData& ModelProto::self_test_data() const { - const ::sentencepiece::SelfTestData* p = self_test_data_; // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.self_test_data) - return p != NULL ? *p : *reinterpret_cast( - &::sentencepiece::_SelfTestData_default_instance_); + return _internal_self_test_data(); +} +inline void ModelProto::unsafe_arena_set_allocated_self_test_data( + ::sentencepiece::SelfTestData* self_test_data) { + if (GetArena() == nullptr) { + delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(self_test_data_); + } + self_test_data_ = self_test_data; + if (self_test_data) { + _has_bits_[0] |= 0x00000004u; + } else { + _has_bits_[0] &= ~0x00000004u; + } + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.ModelProto.self_test_data) } inline ::sentencepiece::SelfTestData* ModelProto::release_self_test_data() { + auto temp = unsafe_arena_release_self_test_data(); + if (GetArena() != nullptr) { + temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); + } + return temp; +} +inline ::sentencepiece::SelfTestData* ModelProto::unsafe_arena_release_self_test_data() { // @@protoc_insertion_point(field_release:sentencepiece.ModelProto.self_test_data) - clear_has_self_test_data(); + _has_bits_[0] &= ~0x00000004u; ::sentencepiece::SelfTestData* temp = self_test_data_; - self_test_data_ = NULL; + self_test_data_ = nullptr; return temp; } -inline ::sentencepiece::SelfTestData* ModelProto::mutable_self_test_data() { - set_has_self_test_data(); - if (self_test_data_ == NULL) { - auto* p = CreateMaybeMessage<::sentencepiece::SelfTestData>(GetArenaNoVirtual()); +inline ::sentencepiece::SelfTestData* ModelProto::_internal_mutable_self_test_data() { + _has_bits_[0] |= 0x00000004u; + if (self_test_data_ == nullptr) { + auto* p = CreateMaybeMessage<::sentencepiece::SelfTestData>(GetArena()); self_test_data_ = p; } - // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.self_test_data) return self_test_data_; } +inline ::sentencepiece::SelfTestData* ModelProto::mutable_self_test_data() { + // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.self_test_data) + return _internal_mutable_self_test_data(); +} inline void ModelProto::set_allocated_self_test_data(::sentencepiece::SelfTestData* self_test_data) { - ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == NULL) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArena(); + if (message_arena == nullptr) { delete self_test_data_; } if (self_test_data) { - ::google::protobuf::Arena* submessage_arena = NULL; + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = + ::PROTOBUF_NAMESPACE_ID::Arena::GetArena(self_test_data); if (message_arena != submessage_arena) { - self_test_data = ::google::protobuf::internal::GetOwnedMessage( + self_test_data = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, self_test_data, submessage_arena); } - set_has_self_test_data(); + _has_bits_[0] |= 0x00000004u; } else { - clear_has_self_test_data(); + _has_bits_[0] &= ~0x00000004u; } self_test_data_ = self_test_data; // @@protoc_insertion_point(field_set_allocated:sentencepiece.ModelProto.self_test_data) } // optional .sentencepiece.NormalizerSpec denormalizer_spec = 5; -inline bool ModelProto::has_denormalizer_spec() const { - return (_has_bits_[0] & 0x00000008u) != 0; +inline bool ModelProto::_internal_has_denormalizer_spec() const { + bool value = (_has_bits_[0] & 0x00000008u) != 0; + PROTOBUF_ASSUME(!value || denormalizer_spec_ != nullptr); + return value; } -inline void ModelProto::set_has_denormalizer_spec() { - _has_bits_[0] |= 0x00000008u; -} -inline void ModelProto::clear_has_denormalizer_spec() { - _has_bits_[0] &= ~0x00000008u; +inline bool ModelProto::has_denormalizer_spec() const { + return _internal_has_denormalizer_spec(); } inline void ModelProto::clear_denormalizer_spec() { - if (denormalizer_spec_ != NULL) denormalizer_spec_->Clear(); - clear_has_denormalizer_spec(); + if (denormalizer_spec_ != nullptr) denormalizer_spec_->Clear(); + _has_bits_[0] &= ~0x00000008u; } inline const ::sentencepiece::NormalizerSpec& ModelProto::_internal_denormalizer_spec() const { - return *denormalizer_spec_; + const ::sentencepiece::NormalizerSpec* p = denormalizer_spec_; + return p != nullptr ? *p : *reinterpret_cast( + &::sentencepiece::_NormalizerSpec_default_instance_); } inline const ::sentencepiece::NormalizerSpec& ModelProto::denormalizer_spec() const { - const ::sentencepiece::NormalizerSpec* p = denormalizer_spec_; // @@protoc_insertion_point(field_get:sentencepiece.ModelProto.denormalizer_spec) - return p != NULL ? *p : *reinterpret_cast( - &::sentencepiece::_NormalizerSpec_default_instance_); + return _internal_denormalizer_spec(); +} +inline void ModelProto::unsafe_arena_set_allocated_denormalizer_spec( + ::sentencepiece::NormalizerSpec* denormalizer_spec) { + if (GetArena() == nullptr) { + delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(denormalizer_spec_); + } + denormalizer_spec_ = denormalizer_spec; + if (denormalizer_spec) { + _has_bits_[0] |= 0x00000008u; + } else { + _has_bits_[0] &= ~0x00000008u; + } + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:sentencepiece.ModelProto.denormalizer_spec) } inline ::sentencepiece::NormalizerSpec* ModelProto::release_denormalizer_spec() { + auto temp = unsafe_arena_release_denormalizer_spec(); + if (GetArena() != nullptr) { + temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); + } + return temp; +} +inline ::sentencepiece::NormalizerSpec* ModelProto::unsafe_arena_release_denormalizer_spec() { // @@protoc_insertion_point(field_release:sentencepiece.ModelProto.denormalizer_spec) - clear_has_denormalizer_spec(); + _has_bits_[0] &= ~0x00000008u; ::sentencepiece::NormalizerSpec* temp = denormalizer_spec_; - denormalizer_spec_ = NULL; + denormalizer_spec_ = nullptr; return temp; } -inline ::sentencepiece::NormalizerSpec* ModelProto::mutable_denormalizer_spec() { - set_has_denormalizer_spec(); - if (denormalizer_spec_ == NULL) { - auto* p = CreateMaybeMessage<::sentencepiece::NormalizerSpec>(GetArenaNoVirtual()); +inline ::sentencepiece::NormalizerSpec* ModelProto::_internal_mutable_denormalizer_spec() { + _has_bits_[0] |= 0x00000008u; + if (denormalizer_spec_ == nullptr) { + auto* p = CreateMaybeMessage<::sentencepiece::NormalizerSpec>(GetArena()); denormalizer_spec_ = p; } - // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.denormalizer_spec) return denormalizer_spec_; } +inline ::sentencepiece::NormalizerSpec* ModelProto::mutable_denormalizer_spec() { + // @@protoc_insertion_point(field_mutable:sentencepiece.ModelProto.denormalizer_spec) + return _internal_mutable_denormalizer_spec(); +} inline void ModelProto::set_allocated_denormalizer_spec(::sentencepiece::NormalizerSpec* denormalizer_spec) { - ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == NULL) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArena(); + if (message_arena == nullptr) { delete denormalizer_spec_; } if (denormalizer_spec) { - ::google::protobuf::Arena* submessage_arena = NULL; + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = + ::PROTOBUF_NAMESPACE_ID::Arena::GetArena(denormalizer_spec); if (message_arena != submessage_arena) { - denormalizer_spec = ::google::protobuf::internal::GetOwnedMessage( + denormalizer_spec = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, denormalizer_spec, submessage_arena); } - set_has_denormalizer_spec(); + _has_bits_[0] |= 0x00000008u; } else { - clear_has_denormalizer_spec(); + _has_bits_[0] &= ~0x00000008u; } denormalizer_spec_ = denormalizer_spec; // @@protoc_insertion_point(field_set_allocated:sentencepiece.ModelProto.denormalizer_spec) @@ -3883,15 +5065,14 @@ inline void ModelProto::set_allocated_denormalizer_spec(::sentencepiece::Normali } // namespace sentencepiece -namespace google { -namespace protobuf { +PROTOBUF_NAMESPACE_OPEN template <> struct is_proto_enum< ::sentencepiece::TrainerSpec_ModelType> : ::std::true_type {}; template <> struct is_proto_enum< ::sentencepiece::ModelProto_SentencePiece_Type> : ::std::true_type {}; -} // namespace protobuf -} // namespace google +PROTOBUF_NAMESPACE_CLOSE // @@protoc_insertion_point(global_scope) -#endif // PROTOBUF_INCLUDED_sentencepiece_5fmodel_2eproto +#include +#endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_sentencepiece_5fmodel_2eproto diff --git a/src/trainer_interface.cc b/src/trainer_interface.cc index eca7c8a7..d340af2f 100644 --- a/src/trainer_interface.cc +++ b/src/trainer_interface.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License.! +#include "trainer_interface.h" + #include #include #include @@ -32,7 +34,6 @@ #include "third_party/absl/strings/str_format.h" #include "third_party/absl/strings/str_join.h" #include "third_party/absl/strings/str_split.h" -#include "trainer_interface.h" #include "unicode_script.h" #include "util.h" @@ -57,16 +58,6 @@ util::Status VerifySpec(const TrainerSpec &trainer_spec) { << "--use_all_vocab=true is valid for WORD/CHAR model."; } - if (trainer_spec.has_mining_sentence_size()) { - LOG(WARNING) - << "--mining_sentence_size() is deprecated. Use --input_sentence_size"; - } - - if (trainer_spec.has_training_sentence_size()) { - LOG(WARNING) << "--training_sentence_size() is deprecated. Use " - "--input_sentence_size"; - } - #define CHECK_RANGE(variable, minval, maxval) \ CHECK_OR_RETURN(variable >= minval && variable <= maxval) diff --git a/third_party/protobuf-lite/arena.cc b/third_party/protobuf-lite/arena.cc index c117c9e5..069dcdfa 100644 --- a/third_party/protobuf-lite/arena.cc +++ b/third_party/protobuf-lite/arena.cc @@ -31,24 +31,26 @@ #include #include +#include #include +#include #ifdef ADDRESS_SANITIZER #include #endif // ADDRESS_SANITIZER -#include +#include -namespace google { static const size_t kMinCleanupListElements = 8; static const size_t kMaxCleanupListElements = 64; // 1kB on 64-bit. +namespace google { namespace protobuf { namespace internal { -std::atomic ArenaImpl::lifecycle_id_generator_; +std::atomic ArenaImpl::lifecycle_id_generator_; #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { static internal::ThreadLocalStorage* thread_cache_ = @@ -57,7 +59,7 @@ ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { } #elif defined(PROTOBUF_USE_DLLS) ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { - static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL }; + static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = {-1, NULL}; return thread_cache_; } #else @@ -125,7 +127,7 @@ ArenaImpl::Block* ArenaImpl::NewBlock(Block* last_block, size_t min_bytes) { ArenaImpl::Block::Block(size_t size, Block* next) : next_(next), pos_(kBlockHeaderSize), size_(size) {} -GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE +PROTOBUF_NOINLINE void ArenaImpl::SerialArena::AddCleanupFallback(void* elem, void (*cleanup)(void*)) { size_t size = cleanup_ ? cleanup_->size * 2 : kMinCleanupListElements; @@ -142,20 +144,10 @@ void ArenaImpl::SerialArena::AddCleanupFallback(void* elem, AddCleanup(elem, cleanup); } -GOOGLE_PROTOBUF_ATTRIBUTE_FUNC_ALIGN(32) -void* ArenaImpl::AllocateAligned(size_t n) { - SerialArena* arena; - if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) { - return arena->AllocateAligned(n); - } else { - return AllocateAlignedFallback(n); - } -} - void* ArenaImpl::AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)) { SerialArena* arena; - if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) { + if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) { return arena->AllocateAlignedAndAddCleanup(n, cleanup); } else { return AllocateAlignedAndAddCleanupFallback(n, cleanup); @@ -164,61 +156,39 @@ void* ArenaImpl::AllocateAlignedAndAddCleanup(size_t n, void ArenaImpl::AddCleanup(void* elem, void (*cleanup)(void*)) { SerialArena* arena; - if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) { + if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) { arena->AddCleanup(elem, cleanup); } else { return AddCleanupFallback(elem, cleanup); } } -GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE +PROTOBUF_NOINLINE void* ArenaImpl::AllocateAlignedFallback(size_t n) { return GetSerialArena()->AllocateAligned(n); } -GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE +PROTOBUF_NOINLINE void* ArenaImpl::AllocateAlignedAndAddCleanupFallback(size_t n, void (*cleanup)(void*)) { return GetSerialArena()->AllocateAlignedAndAddCleanup(n, cleanup); } -GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE +PROTOBUF_NOINLINE void ArenaImpl::AddCleanupFallback(void* elem, void (*cleanup)(void*)) { GetSerialArena()->AddCleanup(elem, cleanup); } -inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -bool ArenaImpl::GetSerialArenaFast(ArenaImpl::SerialArena** arena) { - // If this thread already owns a block in this arena then try to use that. - // This fast path optimizes the case where multiple threads allocate from the - // same arena. - ThreadCache* tc = &thread_cache(); - if (GOOGLE_PREDICT_TRUE(tc->last_lifecycle_id_seen == lifecycle_id_)) { - *arena = tc->last_serial_arena; - return true; - } - - // Check whether we own the last accessed SerialArena on this arena. This - // fast path optimizes the case where a single thread uses multiple arenas. - SerialArena* serial = hint_.load(std::memory_order_acquire); - if (GOOGLE_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) { - *arena = serial; - return true; - } - - return false; -} - ArenaImpl::SerialArena* ArenaImpl::GetSerialArena() { SerialArena* arena; - if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) { + if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) { return arena; } else { return GetSerialArenaFallback(&thread_cache()); } } -GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE +PROTOBUF_NOINLINE void* ArenaImpl::SerialArena::AllocateAlignedFallback(size_t n) { // Sync back to current's pos. head_->set_pos(head_->size() - (limit_ - ptr_)); @@ -241,7 +211,7 @@ uint64 ArenaImpl::SpaceAllocated() const { uint64 ArenaImpl::SpaceUsed() const { SerialArena* serial = threads_.load(std::memory_order_acquire); uint64 space_used = 0; - for ( ; serial; serial = serial->next()) { + for (; serial; serial = serial->next()) { space_used += serial->SpaceUsed(); } return space_used; @@ -285,7 +255,7 @@ uint64 ArenaImpl::SerialArena::Free(ArenaImpl::SerialArena* serial, // We have to be careful in this function, since we will be freeing the Block // that contains this SerialArena. Be careful about accessing |serial|. - for (Block* b = serial->head_; b; ) { + for (Block* b = serial->head_; b;) { // This is inside the block we are freeing, so we need to read it now. Block* next_block = b->next(); space_allocated += (b->size()); @@ -311,7 +281,7 @@ void ArenaImpl::CleanupList() { // properly synchronize Reset() or the destructor will throw a TSAN warning. SerialArena* serial = threads_.load(std::memory_order_relaxed); - for ( ; serial; serial = serial->next()) { + for (; serial; serial = serial->next()) { serial->CleanupList(); } } @@ -323,24 +293,22 @@ void ArenaImpl::SerialArena::CleanupList() { } void ArenaImpl::SerialArena::CleanupListFallback() { - // Cleanup newest chunk: ptrs give us length. + // The first chunk might be only partially full, so calculate its size + // from cleanup_ptr_. Subsequent chunks are always full, so use list->size. size_t n = cleanup_ptr_ - &cleanup_->nodes[0]; - CleanupNode* node = cleanup_ptr_; - for (size_t i = 0; i < n; i++) { - --node; - node->cleanup(node->elem); - } - - // Cleanup older chunks, which are known to be full. - CleanupChunk* list = cleanup_->next; - while (list) { - size_t n = list->size; - CleanupNode* node = &list->nodes[list->size]; - for (size_t i = 0; i < n; i++) { - --node; - node->cleanup(node->elem); + CleanupChunk* list = cleanup_; + while (true) { + CleanupNode* node = &list->nodes[0]; + // Cleanup newest elements first (allocated last). + for (size_t i = n; i > 0; i--) { + node[i - 1].cleanup(node[i - 1].elem); } list = list->next; + if (list == nullptr) { + break; + } + // All but the first chunk are always full. + n = list->size; } } @@ -362,11 +330,11 @@ ArenaImpl::SerialArena* ArenaImpl::SerialArena::New(Block* b, void* owner, return serial; } -GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE +PROTOBUF_NOINLINE ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) { // Look for this SerialArena in our linked list. SerialArena* serial = threads_.load(std::memory_order_acquire); - for ( ; serial; serial = serial->next()) { + for (; serial; serial = serial->next()) { if (serial->owner() == me) { break; } @@ -391,6 +359,11 @@ ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) { } // namespace internal +PROTOBUF_FUNC_ALIGN(32) +void* Arena::AllocateAlignedNoHook(size_t n) { + return impl_.AllocateAligned(n); +} + void Arena::CallDestructorHooks() { uint64 space_allocated = impl_.SpaceAllocated(); // Call the reset hook diff --git a/third_party/protobuf-lite/bytestream.cc b/third_party/protobuf-lite/bytestream.cc index f4af6a50..a0f298ed 100644 --- a/third_party/protobuf-lite/bytestream.cc +++ b/third_party/protobuf-lite/bytestream.cc @@ -33,6 +33,8 @@ #include #include +#include + namespace google { namespace protobuf { namespace strings { @@ -113,7 +115,7 @@ char* GrowingArrayByteSink::GetBuffer(size_t* nbytes) { ShrinkToFit(); char* b = buf_; *nbytes = size_; - buf_ = NULL; + buf_ = nullptr; size_ = capacity_ = 0; return b; } diff --git a/third_party/protobuf-lite/coded_stream.cc b/third_party/protobuf-lite/coded_stream.cc index 0851ff0c..59d86f98 100644 --- a/third_party/protobuf-lite/coded_stream.cc +++ b/third_party/protobuf-lite/coded_stream.cc @@ -38,17 +38,24 @@ // will not cross the end of the buffer, since we can avoid a lot // of branching in this case. -#include +#include + +#include + #include +#include #include -#include -#include -#include + #include #include +#include +#include +#include #include +#include + namespace google { namespace protobuf { namespace io { @@ -59,8 +66,8 @@ static const int kMaxVarintBytes = 10; static const int kMaxVarint32Bytes = 5; -inline bool NextNonEmpty(ZeroCopyInputStream* input, - const void** data, int* size) { +inline bool NextNonEmpty(ZeroCopyInputStream* input, const void** data, + int* size) { bool success; do { success = input->Next(data, size); @@ -82,10 +89,6 @@ CodedInputStream::~CodedInputStream() { int CodedInputStream::default_recursion_limit_ = 100; -void CodedOutputStream::EnableAliasing(bool enabled) { - aliasing_enabled_ = enabled && output_->AllowsAliasing(); -} - void CodedInputStream::BackUpInputToCurrentPosition() { int backup_bytes = BufferSize() + buffer_size_after_limit_ + overflow_bytes_; if (backup_bytes > 0) { @@ -121,9 +124,9 @@ CodedInputStream::Limit CodedInputStream::PushLimit(int byte_limit) { // security: byte_limit is possibly evil, so check for negative values // and overflow. Also check that the new requested limit is before the // previous limit; otherwise we continue to enforce the previous limit. - if (GOOGLE_PREDICT_TRUE(byte_limit >= 0 && - byte_limit <= INT_MAX - current_position && - byte_limit < current_limit_ - current_position)) { + if (PROTOBUF_PREDICT_TRUE(byte_limit >= 0 && + byte_limit <= INT_MAX - current_position && + byte_limit < current_limit_ - current_position)) { current_limit_ = current_position + byte_limit; RecomputeBufferLimits(); } @@ -187,11 +190,13 @@ int CodedInputStream::BytesUntilTotalBytesLimit() const { } void CodedInputStream::PrintTotalBytesLimitError() { - GOOGLE_LOG(ERROR) << "A protocol message was rejected because it was too " - "big (more than " << total_bytes_limit_ - << " bytes). To increase the limit (or to disable these " - "warnings), see CodedInputStream::SetTotalBytesLimit() " - "in google/protobuf/io/coded_stream.h."; + GOOGLE_LOG(ERROR) + << "A protocol message was rejected because it was too " + "big (more than " + << total_bytes_limit_ + << " bytes). To increase the limit (or to disable these " + "warnings), see CodedInputStream::SetTotalBytesLimit() " + "in third_party/protobuf/src/google/protobuf/io/coded_stream.h."; } bool CodedInputStream::SkipFallback(int count, int original_buffer_size) { @@ -234,15 +239,42 @@ bool CodedInputStream::GetDirectBufferPointer(const void** data, int* size) { } bool CodedInputStream::ReadRaw(void* buffer, int size) { - return InternalReadRawInline(buffer, size); + int current_buffer_size; + while ((current_buffer_size = BufferSize()) < size) { + // Reading past end of buffer. Copy what we have, then refresh. + memcpy(buffer, buffer_, current_buffer_size); + buffer = reinterpret_cast(buffer) + current_buffer_size; + size -= current_buffer_size; + Advance(current_buffer_size); + if (!Refresh()) return false; + } + + memcpy(buffer, buffer_, size); + Advance(size); + + return true; } -bool CodedInputStream::ReadString(string* buffer, int size) { +bool CodedInputStream::ReadString(std::string* buffer, int size) { if (size < 0) return false; // security: size is often user-supplied - return InternalReadStringInline(buffer, size); + + if (BufferSize() >= size) { + STLStringResizeUninitialized(buffer, size); + std::pair z = as_string_data(buffer); + if (z.second) { + // Oddly enough, memcpy() requires its first two args to be non-NULL even + // if we copy 0 bytes. So, we have ensured that z.first is non-NULL here. + GOOGLE_DCHECK(z.first != NULL); + memcpy(z.first, buffer_, size); + Advance(size); + } + return true; + } + + return ReadStringFallback(buffer, size); } -bool CodedInputStream::ReadStringFallback(string* buffer, int size) { +bool CodedInputStream::ReadStringFallback(std::string* buffer, int size) { if (!buffer->empty()) { buffer->clear(); } @@ -312,14 +344,28 @@ bool CodedInputStream::ReadLittleEndian64Fallback(uint64* value) { namespace { +// Decodes varint64 with known size, N, and returns next pointer. Knowing N at +// compile time, compiler can generate optimal code. For example, instead of +// subtracting 0x80 at each iteration, it subtracts properly shifted mask once. +template +const uint8* DecodeVarint64KnownSize(const uint8* buffer, uint64* value) { + GOOGLE_DCHECK_GT(N, 0); + uint64 result = static_cast(buffer[N - 1]) << (7 * (N - 1)); + for (int i = 0, offset = 0; i < N - 1; i++, offset += 7) { + result += static_cast(buffer[i] - 0x80) << offset; + } + *value = result; + return buffer + N; +} + // Read a varint from the given buffer, write it to *value, and return a pair. // The first part of the pair is true iff the read was successful. The second // part is buffer + (number of bytes read). This function is always inlined, // so returning a pair is costless. -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -::std::pair ReadVarint32FromArray( - uint32 first_byte, const uint8* buffer, - uint32* value); +PROTOBUF_ALWAYS_INLINE +::std::pair ReadVarint32FromArray(uint32 first_byte, + const uint8* buffer, + uint32* value); inline ::std::pair ReadVarint32FromArray( uint32 first_byte, const uint8* buffer, uint32* value) { // Fast path: We have enough bytes left in the buffer to guarantee that @@ -330,71 +376,72 @@ inline ::std::pair ReadVarint32FromArray( uint32 b; uint32 result = first_byte - 0x80; ++ptr; // We just processed the first byte. Move on to the second. - b = *(ptr++); result += b << 7; if (!(b & 0x80)) goto done; + b = *(ptr++); + result += b << 7; + if (!(b & 0x80)) goto done; result -= 0x80 << 7; - b = *(ptr++); result += b << 14; if (!(b & 0x80)) goto done; + b = *(ptr++); + result += b << 14; + if (!(b & 0x80)) goto done; result -= 0x80 << 14; - b = *(ptr++); result += b << 21; if (!(b & 0x80)) goto done; + b = *(ptr++); + result += b << 21; + if (!(b & 0x80)) goto done; result -= 0x80 << 21; - b = *(ptr++); result += b << 28; if (!(b & 0x80)) goto done; + b = *(ptr++); + result += b << 28; + if (!(b & 0x80)) goto done; // "result -= 0x80 << 28" is irrevelant. // If the input is larger than 32 bits, we still need to read it all // and discard the high-order bits. for (int i = 0; i < kMaxVarintBytes - kMaxVarint32Bytes; i++) { - b = *(ptr++); if (!(b & 0x80)) goto done; + b = *(ptr++); + if (!(b & 0x80)) goto done; } // We have overrun the maximum size of a varint (10 bytes). Assume // the data is corrupt. return std::make_pair(false, ptr); - done: +done: *value = result; return std::make_pair(true, ptr); } -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE::std::pair -ReadVarint64FromArray(const uint8* buffer, uint64* value); +PROTOBUF_ALWAYS_INLINE::std::pair ReadVarint64FromArray( + const uint8* buffer, uint64* value); inline ::std::pair ReadVarint64FromArray( const uint8* buffer, uint64* value) { - const uint8* ptr = buffer; - uint32 b; - - // Splitting into 32-bit pieces gives better performance on 32-bit - // processors. - uint32 part0 = 0, part1 = 0, part2 = 0; - - b = *(ptr++); part0 = b ; if (!(b & 0x80)) goto done; - part0 -= 0x80; - b = *(ptr++); part0 += b << 7; if (!(b & 0x80)) goto done; - part0 -= 0x80 << 7; - b = *(ptr++); part0 += b << 14; if (!(b & 0x80)) goto done; - part0 -= 0x80 << 14; - b = *(ptr++); part0 += b << 21; if (!(b & 0x80)) goto done; - part0 -= 0x80 << 21; - b = *(ptr++); part1 = b ; if (!(b & 0x80)) goto done; - part1 -= 0x80; - b = *(ptr++); part1 += b << 7; if (!(b & 0x80)) goto done; - part1 -= 0x80 << 7; - b = *(ptr++); part1 += b << 14; if (!(b & 0x80)) goto done; - part1 -= 0x80 << 14; - b = *(ptr++); part1 += b << 21; if (!(b & 0x80)) goto done; - part1 -= 0x80 << 21; - b = *(ptr++); part2 = b ; if (!(b & 0x80)) goto done; - part2 -= 0x80; - b = *(ptr++); part2 += b << 7; if (!(b & 0x80)) goto done; - // "part2 -= 0x80 << 7" is irrelevant because (0x80 << 7) << 56 is 0. - - // We have overrun the maximum size of a varint (10 bytes). Assume - // the data is corrupt. - return std::make_pair(false, ptr); + // Assumes varint64 is at least 2 bytes. + GOOGLE_DCHECK_GE(buffer[0], 128); + + const uint8* next; + if (buffer[1] < 128) { + next = DecodeVarint64KnownSize<2>(buffer, value); + } else if (buffer[2] < 128) { + next = DecodeVarint64KnownSize<3>(buffer, value); + } else if (buffer[3] < 128) { + next = DecodeVarint64KnownSize<4>(buffer, value); + } else if (buffer[4] < 128) { + next = DecodeVarint64KnownSize<5>(buffer, value); + } else if (buffer[5] < 128) { + next = DecodeVarint64KnownSize<6>(buffer, value); + } else if (buffer[6] < 128) { + next = DecodeVarint64KnownSize<7>(buffer, value); + } else if (buffer[7] < 128) { + next = DecodeVarint64KnownSize<8>(buffer, value); + } else if (buffer[8] < 128) { + next = DecodeVarint64KnownSize<9>(buffer, value); + } else if (buffer[9] < 128) { + next = DecodeVarint64KnownSize<10>(buffer, value); + } else { + // We have overrun the maximum size of a varint (10 bytes). Assume + // the data is corrupt. + return std::make_pair(false, buffer + 11); + } - done: - *value = (static_cast(part0)) | - (static_cast(part1) << 28) | - (static_cast(part2) << 56); - return std::make_pair(true, ptr); + return std::make_pair(true, next); } } // namespace @@ -416,7 +463,7 @@ int64 CodedInputStream::ReadVarint32Fallback(uint32 first_byte_or_zero) { << "Caller should provide us with *buffer_ when buffer is non-empty"; uint32 temp; ::std::pair p = - ReadVarint32FromArray(first_byte_or_zero, buffer_, &temp); + ReadVarint32FromArray(first_byte_or_zero, buffer_, &temp); if (!p.first) return -1; buffer_ = p.second; return temp; @@ -619,156 +666,285 @@ bool CodedInputStream::Refresh() { // CodedOutputStream ================================================= -std::atomic CodedOutputStream::default_serialization_deterministic_{ - false}; +void EpsCopyOutputStream::EnableAliasing(bool enabled) { + aliasing_enabled_ = enabled && stream_->AllowsAliasing(); +} -CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output) - : CodedOutputStream(output, true) {} +int64 EpsCopyOutputStream::ByteCount(uint8* ptr) const { + // Calculate the current offset relative to the end of the stream buffer. + int delta = (end_ - ptr) + (buffer_end_ ? 0 : kSlopBytes); + return stream_->ByteCount() - delta; +} -CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output, - bool do_eager_refresh) - : output_(output), - buffer_(NULL), - buffer_size_(0), - total_bytes_(0), - had_error_(false), - aliasing_enabled_(false), - is_serialization_deterministic_(IsDefaultSerializationDeterministic()) { - if (do_eager_refresh) { - // Eagerly Refresh() so buffer space is immediately available. - Refresh(); - // The Refresh() may have failed. If the client doesn't write any data, - // though, don't consider this an error. If the client does write data, then - // another Refresh() will be attempted and it will set the error once again. - had_error_ = false; +// Flushes what's written out to the underlying ZeroCopyOutputStream buffers. +// Returns the size remaining in the buffer and sets buffer_end_ to the start +// of the remaining buffer, ie. [buffer_end_, buffer_end_ + return value) +int EpsCopyOutputStream::Flush(uint8* ptr) { + while (buffer_end_ && ptr > end_) { + int overrun = ptr - end_; + GOOGLE_DCHECK(!had_error_); + GOOGLE_DCHECK(overrun <= kSlopBytes); // NOLINT + ptr = Next() + overrun; + if (had_error_) return 0; } + int s; + if (buffer_end_) { + std::memcpy(buffer_end_, buffer_, ptr - buffer_); + buffer_end_ += ptr - buffer_; + s = end_ - ptr; + } else { + // The stream is writing directly in the ZeroCopyOutputStream buffer. + s = end_ + kSlopBytes - ptr; + buffer_end_ = ptr; + } + GOOGLE_DCHECK(s >= 0); // NOLINT + return s; } -CodedOutputStream::~CodedOutputStream() { - Trim(); +uint8* EpsCopyOutputStream::Trim(uint8* ptr) { + if (had_error_) return ptr; + int s = Flush(ptr); + if (s) stream_->BackUp(s); + // Reset to initial state (expecting new buffer) + buffer_end_ = end_ = buffer_; + return buffer_; } -void CodedOutputStream::Trim() { - if (buffer_size_ > 0) { - output_->BackUp(buffer_size_); - total_bytes_ -= buffer_size_; - buffer_size_ = 0; - buffer_ = NULL; - } + +uint8* EpsCopyOutputStream::FlushAndResetBuffer(uint8* ptr) { + if (had_error_) return buffer_; + int s = Flush(ptr); + if (had_error_) return buffer_; + return SetInitialBuffer(buffer_end_, s); } -bool CodedOutputStream::Skip(int count) { +bool EpsCopyOutputStream::Skip(int count, uint8** pp) { if (count < 0) return false; - - while (count > buffer_size_) { - count -= buffer_size_; - if (!Refresh()) return false; + if (had_error_) { + *pp = buffer_; + return false; } - - Advance(count); + int size = Flush(*pp); + if (had_error_) { + *pp = buffer_; + return false; + } + void* data = buffer_end_; + while (count > size) { + count -= size; + if (!stream_->Next(&data, &size)) { + *pp = Error(); + return false; + } + } + *pp = SetInitialBuffer(static_cast(data) + count, size - count); return true; } -bool CodedOutputStream::GetDirectBufferPointer(void** data, int* size) { - if (buffer_size_ == 0 && !Refresh()) return false; - - *data = buffer_; - *size = buffer_size_; +bool EpsCopyOutputStream::GetDirectBufferPointer(void** data, int* size, + uint8** pp) { + if (had_error_) { + *pp = buffer_; + return false; + } + *size = Flush(*pp); + if (had_error_) { + *pp = buffer_; + return false; + } + *data = buffer_end_; + while (*size == 0) { + if (!stream_->Next(data, size)) { + *pp = Error(); + return false; + } + } + *pp = SetInitialBuffer(*data, *size); return true; } -void CodedOutputStream::WriteRaw(const void* data, int size) { - while (buffer_size_ < size) { - memcpy(buffer_, data, buffer_size_); - size -= buffer_size_; - data = reinterpret_cast(data) + buffer_size_; - if (!Refresh()) return; +uint8* EpsCopyOutputStream::GetDirectBufferForNBytesAndAdvance(int size, + uint8** pp) { + if (had_error_) { + *pp = buffer_; + return nullptr; + } + int s = Flush(*pp); + if (had_error_) { + *pp = buffer_; + return nullptr; + } + if (s >= size) { + auto res = buffer_end_; + *pp = SetInitialBuffer(buffer_end_ + size, s - size); + return res; + } else { + *pp = SetInitialBuffer(buffer_end_, s); + return nullptr; } - - memcpy(buffer_, data, size); - Advance(size); -} - -uint8* CodedOutputStream::WriteRawToArray( - const void* data, int size, uint8* target) { - memcpy(target, data, size); - return target + size; } - -void CodedOutputStream::WriteAliasedRaw(const void* data, int size) { - if (size < buffer_size_ - ) { - WriteRaw(data, size); +uint8* EpsCopyOutputStream::Next() { + GOOGLE_DCHECK(!had_error_); // NOLINT + if (PROTOBUF_PREDICT_FALSE(stream_ == nullptr)) return Error(); + if (buffer_end_) { + // We're in the patch buffer and need to fill up the previous buffer. + std::memcpy(buffer_end_, buffer_, end_ - buffer_); + uint8* ptr; + int size; + do { + void* data; + if (PROTOBUF_PREDICT_FALSE(!stream_->Next(&data, &size))) { + // Stream has an error, we use the patch buffer to continue to be + // able to write. + return Error(); + } + ptr = static_cast(data); + } while (size == 0); + if (PROTOBUF_PREDICT_TRUE(size > kSlopBytes)) { + std::memcpy(ptr, end_, kSlopBytes); + end_ = ptr + size - kSlopBytes; + buffer_end_ = nullptr; + return ptr; + } else { + GOOGLE_DCHECK(size > 0); // NOLINT + // Buffer to small + std::memmove(buffer_, end_, kSlopBytes); + buffer_end_ = ptr; + end_ = buffer_ + size; + return buffer_; + } } else { - Trim(); - - total_bytes_ += size; - had_error_ |= !output_->WriteAliasedRaw(data, size); + std::memcpy(buffer_, end_, kSlopBytes); + buffer_end_ = end_; + end_ = buffer_ + kSlopBytes; + return buffer_; } } -void CodedOutputStream::WriteLittleEndian32(uint32 value) { - uint8 bytes[sizeof(value)]; - - bool use_fast = buffer_size_ >= sizeof(value); - uint8* ptr = use_fast ? buffer_ : bytes; - - WriteLittleEndian32ToArray(value, ptr); +uint8* EpsCopyOutputStream::EnsureSpaceFallback(uint8* ptr) { + do { + if (PROTOBUF_PREDICT_FALSE(had_error_)) return buffer_; + int overrun = ptr - end_; + GOOGLE_DCHECK(overrun >= 0); // NOLINT + GOOGLE_DCHECK(overrun <= kSlopBytes); // NOLINT + ptr = Next() + overrun; + } while (ptr >= end_); + GOOGLE_DCHECK(ptr < end_); // NOLINT + return ptr; +} + +uint8* EpsCopyOutputStream::WriteRawFallback(const void* data, int size, + uint8* ptr) { + int s = GetSize(ptr); + while (s < size) { + std::memcpy(ptr, data, s); + size -= s; + data = static_cast(data) + s; + ptr = EnsureSpaceFallback(ptr + s); + s = GetSize(ptr); + } + std::memcpy(ptr, data, size); + return ptr + size; +} - if (use_fast) { - Advance(sizeof(value)); +uint8* EpsCopyOutputStream::WriteAliasedRaw(const void* data, int size, + uint8* ptr) { + if (size < GetSize(ptr) + ) { + return WriteRaw(data, size, ptr); } else { - WriteRaw(bytes, sizeof(value)); + ptr = Trim(ptr); + if (stream_->WriteAliasedRaw(data, size)) return ptr; + return Error(); } } -void CodedOutputStream::WriteLittleEndian64(uint64 value) { - uint8 bytes[sizeof(value)]; - - bool use_fast = buffer_size_ >= sizeof(value); - uint8* ptr = use_fast ? buffer_ : bytes; - - WriteLittleEndian64ToArray(value, ptr); - - if (use_fast) { - Advance(sizeof(value)); - } else { - WriteRaw(bytes, sizeof(value)); +#ifndef PROTOBUF_LITTLE_ENDIAN +uint8* EpsCopyOutputStream::WriteRawLittleEndian32(const void* data, int size, + uint8* ptr) { + auto p = static_cast(data); + auto end = p + size; + while (end - p >= kSlopBytes) { + ptr = EnsureSpace(ptr); + uint32 buffer[4]; + static_assert(sizeof(buffer) == kSlopBytes, "Buffer must be kSlopBytes"); + std::memcpy(buffer, p, kSlopBytes); + p += kSlopBytes; + for (auto x : buffer) + ptr = CodedOutputStream::WriteLittleEndian32ToArray(x, ptr); + } + while (p < end) { + ptr = EnsureSpace(ptr); + uint32 buffer; + std::memcpy(&buffer, p, 4); + p += 4; + ptr = CodedOutputStream::WriteLittleEndian32ToArray(buffer, ptr); + } + return ptr; +} + +uint8* EpsCopyOutputStream::WriteRawLittleEndian64(const void* data, int size, + uint8* ptr) { + auto p = static_cast(data); + auto end = p + size; + while (end - p >= kSlopBytes) { + ptr = EnsureSpace(ptr); + uint64 buffer[2]; + static_assert(sizeof(buffer) == kSlopBytes, "Buffer must be kSlopBytes"); + std::memcpy(buffer, p, kSlopBytes); + p += kSlopBytes; + for (auto x : buffer) + ptr = CodedOutputStream::WriteLittleEndian64ToArray(x, ptr); } + while (p < end) { + ptr = EnsureSpace(ptr); + uint64 buffer; + std::memcpy(&buffer, p, 8); + p += 8; + ptr = CodedOutputStream::WriteLittleEndian64ToArray(buffer, ptr); + } + return ptr; } +#endif + -void CodedOutputStream::WriteVarint32SlowPath(uint32 value) { - uint8 bytes[kMaxVarint32Bytes]; - uint8* target = &bytes[0]; - uint8* end = WriteVarint32ToArray(value, target); - int size = end - target; - WriteRaw(bytes, size); +uint8* EpsCopyOutputStream::WriteStringMaybeAliasedOutline(uint32 num, + const std::string& s, + uint8* ptr) { + ptr = EnsureSpace(ptr); + uint32 size = s.size(); + ptr = WriteLengthDelim(num, size, ptr); + return WriteRawMaybeAliased(s.data(), size, ptr); } -void CodedOutputStream::WriteVarint64SlowPath(uint64 value) { - uint8 bytes[kMaxVarintBytes]; - uint8* target = &bytes[0]; - uint8* end = WriteVarint64ToArray(value, target); - int size = end - target; - WriteRaw(bytes, size); +uint8* EpsCopyOutputStream::WriteStringOutline(uint32 num, const std::string& s, + uint8* ptr) { + ptr = EnsureSpace(ptr); + uint32 size = s.size(); + ptr = WriteLengthDelim(num, size, ptr); + return WriteRaw(s.data(), size, ptr); } -bool CodedOutputStream::Refresh() { - void* void_buffer; - if (output_->Next(&void_buffer, &buffer_size_)) { - buffer_ = reinterpret_cast(void_buffer); - total_bytes_ += buffer_size_; - return true; - } else { - buffer_ = NULL; - buffer_size_ = 0; - had_error_ = true; - return false; +std::atomic CodedOutputStream::default_serialization_deterministic_{ + false}; + +CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* stream, + bool do_eager_refresh) + : impl_(stream, IsDefaultSerializationDeterministic(), &cur_), + start_count_(stream->ByteCount()) { + if (do_eager_refresh) { + void* data; + int size; + if (!stream->Next(&data, &size) || size == 0) return; + cur_ = impl_.SetInitialBuffer(data, size); } } -uint8* CodedOutputStream::WriteStringWithSizeToArray(const string& str, +CodedOutputStream::~CodedOutputStream() { Trim(); } + + +uint8* CodedOutputStream::WriteStringWithSizeToArray(const std::string& str, uint8* target) { GOOGLE_DCHECK_LE(str.size(), kuint32max); target = WriteVarint32ToArray(str.size(), target); diff --git a/third_party/protobuf-lite/common.cc b/third_party/protobuf-lite/common.cc index 6544c6ed..4db493b7 100644 --- a/third_party/protobuf-lite/common.cc +++ b/third_party/protobuf-lite/common.cc @@ -30,20 +30,18 @@ // Author: kenton@google.com (Kenton Varda) -#include // TODO(gerbens) ideally remove this. #include -#include -#include -#include -#include -#include + +#include #include #include #include #include #ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN // We only need minimal includes +#endif #include #define snprintf _snprintf // see comment in strutil.cc #elif defined(HAVE_PTHREAD) @@ -55,6 +53,16 @@ #include #endif +#include +#include +#include +#include +#include +#include +#include + +#include + namespace google { namespace protobuf { @@ -174,22 +182,7 @@ void NullLogHandler(LogLevel /* level */, const char* /* filename */, } static LogHandler* log_handler_ = &DefaultLogHandler; -static int log_silencer_count_ = 0; - -static Mutex* log_silencer_count_mutex_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(log_silencer_count_init_); - -void DeleteLogSilencerCount() { - delete log_silencer_count_mutex_; - log_silencer_count_mutex_ = NULL; -} -void InitLogSilencerCount() { - log_silencer_count_mutex_ = new Mutex; - OnShutdown(&DeleteLogSilencerCount); -} -void InitLogSilencerCountOnce() { - GoogleOnceInit(&log_silencer_count_init_, &InitLogSilencerCount); -} +static std::atomic log_silencer_count_ = ATOMIC_VAR_INIT(0); LogMessage& LogMessage::operator<<(const string& value) { message_ += value; @@ -206,8 +199,7 @@ LogMessage& LogMessage::operator<<(const StringPiece& value) { return *this; } -LogMessage& LogMessage::operator<<( - const ::google::protobuf::util::Status& status) { +LogMessage& LogMessage::operator<<(const util::Status& status) { message_ += status.ToString(); return *this; } @@ -243,8 +235,8 @@ DECLARE_STREAM_OPERATOR(long , "%ld") DECLARE_STREAM_OPERATOR(unsigned long, "%lu") DECLARE_STREAM_OPERATOR(double , "%g" ) DECLARE_STREAM_OPERATOR(void* , "%p" ) -DECLARE_STREAM_OPERATOR(long long , "%" GOOGLE_LL_FORMAT "d") -DECLARE_STREAM_OPERATOR(unsigned long long, "%" GOOGLE_LL_FORMAT "u") +DECLARE_STREAM_OPERATOR(long long , "%" PROTOBUF_LL_FORMAT "d") +DECLARE_STREAM_OPERATOR(unsigned long long, "%" PROTOBUF_LL_FORMAT "u") #undef DECLARE_STREAM_OPERATOR LogMessage::LogMessage(LogLevel level, const char* filename, int line) @@ -255,8 +247,6 @@ void LogMessage::Finish() { bool suppress = false; if (level_ != LOGLEVEL_FATAL) { - InitLogSilencerCountOnce(); - MutexLock lock(log_silencer_count_mutex_); suppress = log_silencer_count_ > 0; } @@ -282,9 +272,9 @@ void LogFinisher::operator=(LogMessage& other) { LogHandler* SetLogHandler(LogHandler* new_func) { LogHandler* old = internal::log_handler_; if (old == &internal::NullLogHandler) { - old = NULL; + old = nullptr; } - if (new_func == NULL) { + if (new_func == nullptr) { internal::log_handler_ = &internal::NullLogHandler; } else { internal::log_handler_ = new_func; @@ -293,14 +283,10 @@ LogHandler* SetLogHandler(LogHandler* new_func) { } LogSilencer::LogSilencer() { - internal::InitLogSilencerCountOnce(); - MutexLock lock(internal::log_silencer_count_mutex_); ++internal::log_silencer_count_; }; LogSilencer::~LogSilencer() { - internal::InitLogSilencerCountOnce(); - MutexLock lock(internal::log_silencer_count_mutex_); --internal::log_silencer_count_; }; @@ -331,52 +317,6 @@ uint32 ghtonl(uint32 x) { return result; } -// =================================================================== -// Shutdown support. - -namespace internal { - -typedef void OnShutdownFunc(); -struct ShutdownData { - ~ShutdownData() { - std::reverse(functions.begin(), functions.end()); - for (auto pair : functions) pair.first(pair.second); - } - - static ShutdownData* get() { - static auto* data = new ShutdownData; - return data; - } - - std::vector> functions; - Mutex mutex; -}; - -static void RunZeroArgFunc(const void* arg) { - reinterpret_cast(const_cast(arg))(); -} - -void OnShutdown(void (*func)()) { - OnShutdownRun(RunZeroArgFunc, reinterpret_cast(func)); -} - -void OnShutdownRun(void (*f)(const void*), const void* arg) { - auto shutdown_data = ShutdownData::get(); - MutexLock lock(&shutdown_data->mutex); - shutdown_data->functions.push_back(std::make_pair(f, arg)); -} - -} // namespace internal - -void ShutdownProtobufLibrary() { - // This function should be called only once, but accepts multiple calls. - static bool is_shutdown = false; - if (!is_shutdown) { - delete internal::ShutdownData::get(); - is_shutdown = true; - } -} - #if PROTOBUF_USE_EXCEPTIONS FatalException::~FatalException() throw() {} diff --git a/third_party/protobuf-lite/extension_set.cc b/third_party/protobuf-lite/extension_set.cc index cb205c4f..3b1441e6 100644 --- a/third_party/protobuf-lite/extension_set.cc +++ b/third_party/protobuf-lite/extension_set.cc @@ -32,16 +32,23 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include +#include + #include +#include #include #include -#include -#include +#include +#include #include -#include +#include +#include +#include #include #include +#include + +#include namespace google { namespace protobuf { @@ -69,38 +76,47 @@ inline bool is_packable(WireFormatLite::WireType type) { case WireFormatLite::WIRETYPE_END_GROUP: return false; - // Do not add a default statement. Let the compiler complain when someone - // adds a new wire type. + // Do not add a default statement. Let the compiler complain when someone + // adds a new wire type. } GOOGLE_LOG(FATAL) << "can't reach here."; return false; } // Registry stuff. -typedef hash_map, - ExtensionInfo> ExtensionRegistry; +struct ExtensionHasher { + std::size_t operator()(const std::pair& p) const { + return std::hash{}(p.first) ^ + std::hash{}(p.second); + } +}; + +typedef std::unordered_map, ExtensionInfo, + ExtensionHasher> + ExtensionRegistry; static const ExtensionRegistry* global_registry = nullptr; // This function is only called at startup, so there is no need for thread- // safety. -void Register(const MessageLite* containing_type, - int number, ExtensionInfo info) { +void Register(const MessageLite* containing_type, int number, + ExtensionInfo info) { static auto local_static_registry = OnShutdownDelete(new ExtensionRegistry); global_registry = local_static_registry; if (!InsertIfNotPresent(local_static_registry, - std::make_pair(containing_type, number), info)) { + std::make_pair(containing_type, number), info)) { GOOGLE_LOG(FATAL) << "Multiple extension registrations for type \"" - << containing_type->GetTypeName() - << "\", field number " << number << "."; + << containing_type->GetTypeName() << "\", field number " + << number << "."; } } -const ExtensionInfo* FindRegisteredExtension( - const MessageLite* containing_type, int number) { +const ExtensionInfo* FindRegisteredExtension(const MessageLite* containing_type, + int number) { return global_registry == nullptr - ? nullptr - : FindOrNull(*global_registry, std::make_pair(containing_type, number)); + ? nullptr + : FindOrNull(*global_registry, + std::make_pair(containing_type, number)); } } // namespace @@ -160,7 +176,7 @@ void ExtensionSet::RegisterMessageExtension(const MessageLite* containing_type, GOOGLE_CHECK(type == WireFormatLite::TYPE_MESSAGE || type == WireFormatLite::TYPE_GROUP); ExtensionInfo info(type, is_repeated, is_packed); - info.message_prototype = prototype; + info.message_info = {prototype}; Register(containing_type, number, info); } @@ -168,34 +184,50 @@ void ExtensionSet::RegisterMessageExtension(const MessageLite* containing_type, // =================================================================== // Constructors and basic methods. -ExtensionSet::ExtensionSet(::google::protobuf::Arena* arena) +ExtensionSet::ExtensionSet(Arena* arena) : arena_(arena), flat_capacity_(0), flat_size_(0), - map_{flat_capacity_ == 0 ? NULL - : ::google::protobuf::Arena::CreateArray( - arena_, flat_capacity_)} {} + map_{flat_capacity_ == 0 + ? NULL + : Arena::CreateArray(arena_, flat_capacity_)} {} ExtensionSet::ExtensionSet() : arena_(NULL), flat_capacity_(0), flat_size_(0), - map_{flat_capacity_ == 0 ? NULL - : ::google::protobuf::Arena::CreateArray( - arena_, flat_capacity_)} {} + map_{flat_capacity_ == 0 + ? NULL + : Arena::CreateArray(arena_, flat_capacity_)} {} ExtensionSet::~ExtensionSet() { // Deletes all allocated extensions. if (arena_ == NULL) { ForEach([](int /* number */, Extension& ext) { ext.Free(); }); - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { delete map_.large; } else { - delete[] map_.flat; + DeleteFlatMap(map_.flat, flat_capacity_); } } } +void ExtensionSet::DeleteFlatMap(const ExtensionSet::KeyValue* flat, + uint16 flat_capacity) { +#ifdef __cpp_sized_deallocation + // Arena::CreateArray already requires a trivially destructible type, but + // ensure this constraint is not violated in the future. + static_assert(std::is_trivially_destructible::value, + "CreateArray requires a trivially destructible type"); + // A const-cast is needed, but this is safe as we are about to deallocate the + // array. + ::operator delete[](const_cast(flat), + sizeof(*flat) * flat_capacity); +#else // !__cpp_sized_deallocation + delete[] flat; +#endif // !__cpp_sized_deallocation +} + // Defined in extension_set_heavy.cc. // void ExtensionSet::AppendToList(const Descriptor* containing_type, // const DescriptorPool* pool, @@ -246,88 +278,88 @@ void ExtensionSet::ClearExtension(int number) { namespace { -enum Cardinality { - REPEATED, - OPTIONAL -}; +enum { REPEATED_FIELD, OPTIONAL_FIELD }; } // namespace -#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE) \ - GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? REPEATED : OPTIONAL, LABEL); \ +#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE) \ + GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? REPEATED_FIELD : OPTIONAL_FIELD, LABEL); \ GOOGLE_DCHECK_EQ(cpp_type((EXTENSION).type), WireFormatLite::CPPTYPE_##CPPTYPE) // ------------------------------------------------------------------- // Primitives -#define PRIMITIVE_ACCESSORS(UPPERCASE, LOWERCASE, CAMELCASE) \ - \ -LOWERCASE ExtensionSet::Get##CAMELCASE(int number, \ - LOWERCASE default_value) const { \ - const Extension* extension = FindOrNull(number); \ - if (extension == NULL || extension->is_cleared) { \ - return default_value; \ - } else { \ - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, UPPERCASE); \ - return extension->LOWERCASE##_value; \ - } \ -} \ - \ -void ExtensionSet::Set##CAMELCASE(int number, FieldType type, \ - LOWERCASE value, \ - const FieldDescriptor* descriptor) { \ - Extension* extension; \ - if (MaybeNewExtension(number, descriptor, &extension)) { \ - extension->type = type; \ - GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_##UPPERCASE); \ - extension->is_repeated = false; \ - } else { \ - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, UPPERCASE); \ - } \ - extension->is_cleared = false; \ - extension->LOWERCASE##_value = value; \ -} \ - \ -LOWERCASE ExtensionSet::GetRepeated##CAMELCASE(int number, int index) const { \ - const Extension* extension = FindOrNull(number); \ - GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; \ - GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE); \ - return extension->repeated_##LOWERCASE##_value->Get(index); \ -} \ - \ -void ExtensionSet::SetRepeated##CAMELCASE( \ - int number, int index, LOWERCASE value) { \ - Extension* extension = FindOrNull(number); \ - GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; \ - GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE); \ - extension->repeated_##LOWERCASE##_value->Set(index, value); \ -} \ - \ -void ExtensionSet::Add##CAMELCASE(int number, FieldType type, \ - bool packed, LOWERCASE value, \ - const FieldDescriptor* descriptor) { \ - Extension* extension; \ - if (MaybeNewExtension(number, descriptor, &extension)) { \ - extension->type = type; \ - GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_##UPPERCASE); \ - extension->is_repeated = true; \ - extension->is_packed = packed; \ - extension->repeated_##LOWERCASE##_value = \ - Arena::CreateMessage >(arena_); \ - } else { \ - GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE); \ - GOOGLE_DCHECK_EQ(extension->is_packed, packed); \ - } \ - extension->repeated_##LOWERCASE##_value->Add(value); \ -} - -PRIMITIVE_ACCESSORS( INT32, int32, Int32) -PRIMITIVE_ACCESSORS( INT64, int64, Int64) +#define PRIMITIVE_ACCESSORS(UPPERCASE, LOWERCASE, CAMELCASE) \ + \ + LOWERCASE ExtensionSet::Get##CAMELCASE(int number, LOWERCASE default_value) \ + const { \ + const Extension* extension = FindOrNull(number); \ + if (extension == NULL || extension->is_cleared) { \ + return default_value; \ + } else { \ + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, UPPERCASE); \ + return extension->LOWERCASE##_value; \ + } \ + } \ + \ + void ExtensionSet::Set##CAMELCASE(int number, FieldType type, \ + LOWERCASE value, \ + const FieldDescriptor* descriptor) { \ + Extension* extension; \ + if (MaybeNewExtension(number, descriptor, &extension)) { \ + extension->type = type; \ + GOOGLE_DCHECK_EQ(cpp_type(extension->type), \ + WireFormatLite::CPPTYPE_##UPPERCASE); \ + extension->is_repeated = false; \ + } else { \ + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, UPPERCASE); \ + } \ + extension->is_cleared = false; \ + extension->LOWERCASE##_value = value; \ + } \ + \ + LOWERCASE ExtensionSet::GetRepeated##CAMELCASE(int number, int index) \ + const { \ + const Extension* extension = FindOrNull(number); \ + GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; \ + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE); \ + return extension->repeated_##LOWERCASE##_value->Get(index); \ + } \ + \ + void ExtensionSet::SetRepeated##CAMELCASE(int number, int index, \ + LOWERCASE value) { \ + Extension* extension = FindOrNull(number); \ + GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; \ + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE); \ + extension->repeated_##LOWERCASE##_value->Set(index, value); \ + } \ + \ + void ExtensionSet::Add##CAMELCASE(int number, FieldType type, bool packed, \ + LOWERCASE value, \ + const FieldDescriptor* descriptor) { \ + Extension* extension; \ + if (MaybeNewExtension(number, descriptor, &extension)) { \ + extension->type = type; \ + GOOGLE_DCHECK_EQ(cpp_type(extension->type), \ + WireFormatLite::CPPTYPE_##UPPERCASE); \ + extension->is_repeated = true; \ + extension->is_packed = packed; \ + extension->repeated_##LOWERCASE##_value = \ + Arena::CreateMessage>(arena_); \ + } else { \ + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE); \ + GOOGLE_DCHECK_EQ(extension->is_packed, packed); \ + } \ + extension->repeated_##LOWERCASE##_value->Add(value); \ + } + +PRIMITIVE_ACCESSORS(INT32, int32, Int32) +PRIMITIVE_ACCESSORS(INT64, int64, Int64) PRIMITIVE_ACCESSORS(UINT32, uint32, UInt32) PRIMITIVE_ACCESSORS(UINT64, uint64, UInt64) -PRIMITIVE_ACCESSORS( FLOAT, float, Float) +PRIMITIVE_ACCESSORS(FLOAT, float, Float) PRIMITIVE_ACCESSORS(DOUBLE, double, Double) -PRIMITIVE_ACCESSORS( BOOL, bool, Bool) +PRIMITIVE_ACCESSORS(BOOL, bool, Bool) #undef PRIMITIVE_ACCESSORS @@ -358,43 +390,43 @@ void* ExtensionSet::MutableRawRepeatedField(int number, FieldType field_type, static_cast(field_type))) { case WireFormatLite::CPPTYPE_INT32: extension->repeated_int32_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_INT64: extension->repeated_int64_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_UINT32: extension->repeated_uint32_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_UINT64: extension->repeated_uint64_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_DOUBLE: extension->repeated_double_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_FLOAT: extension->repeated_float_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_BOOL: extension->repeated_bool_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_ENUM: extension->repeated_enum_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_STRING: extension->repeated_string_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; case WireFormatLite::CPPTYPE_MESSAGE: extension->repeated_message_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); break; } } @@ -414,7 +446,6 @@ void* ExtensionSet::MutableRawRepeatedField(int number) { return extension->repeated_int32_value; } - // ------------------------------------------------------------------- // Enums @@ -424,7 +455,7 @@ int ExtensionSet::GetEnum(int number, int default_value) const { // Not present. Return the default value. return default_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, ENUM); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, ENUM); return extension->enum_value; } } @@ -437,7 +468,7 @@ void ExtensionSet::SetEnum(int number, FieldType type, int value, GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_ENUM); extension->is_repeated = false; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, ENUM); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, ENUM); } extension->is_cleared = false; extension->enum_value = value; @@ -446,19 +477,18 @@ void ExtensionSet::SetEnum(int number, FieldType type, int value, int ExtensionSet::GetRepeatedEnum(int number, int index) const { const Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM); return extension->repeated_enum_value->Get(index); } void ExtensionSet::SetRepeatedEnum(int number, int index, int value) { Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM); extension->repeated_enum_value->Set(index, value); } -void ExtensionSet::AddEnum(int number, FieldType type, - bool packed, int value, +void ExtensionSet::AddEnum(int number, FieldType type, bool packed, int value, const FieldDescriptor* descriptor) { Extension* extension; if (MaybeNewExtension(number, descriptor, &extension)) { @@ -467,9 +497,9 @@ void ExtensionSet::AddEnum(int number, FieldType type, extension->is_repeated = true; extension->is_packed = packed; extension->repeated_enum_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM); GOOGLE_DCHECK_EQ(extension->is_packed, packed); } extension->repeated_enum_value->Add(value); @@ -478,49 +508,50 @@ void ExtensionSet::AddEnum(int number, FieldType type, // ------------------------------------------------------------------- // Strings -const string& ExtensionSet::GetString(int number, - const string& default_value) const { +const std::string& ExtensionSet::GetString( + int number, const std::string& default_value) const { const Extension* extension = FindOrNull(number); if (extension == NULL || extension->is_cleared) { // Not present. Return the default value. return default_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, STRING); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, STRING); return *extension->string_value; } } -string* ExtensionSet::MutableString(int number, FieldType type, - const FieldDescriptor* descriptor) { +std::string* ExtensionSet::MutableString(int number, FieldType type, + const FieldDescriptor* descriptor) { Extension* extension; if (MaybeNewExtension(number, descriptor, &extension)) { extension->type = type; GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_STRING); extension->is_repeated = false; - extension->string_value = Arena::Create(arena_); + extension->string_value = Arena::Create(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, STRING); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, STRING); } extension->is_cleared = false; return extension->string_value; } -const string& ExtensionSet::GetRepeatedString(int number, int index) const { +const std::string& ExtensionSet::GetRepeatedString(int number, + int index) const { const Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING); return extension->repeated_string_value->Get(index); } -string* ExtensionSet::MutableRepeatedString(int number, int index) { +std::string* ExtensionSet::MutableRepeatedString(int number, int index) { Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING); return extension->repeated_string_value->Mutable(index); } -string* ExtensionSet::AddString(int number, FieldType type, - const FieldDescriptor* descriptor) { +std::string* ExtensionSet::AddString(int number, FieldType type, + const FieldDescriptor* descriptor) { Extension* extension; if (MaybeNewExtension(number, descriptor, &extension)) { extension->type = type; @@ -528,9 +559,9 @@ string* ExtensionSet::AddString(int number, FieldType type, extension->is_repeated = true; extension->is_packed = false; extension->repeated_string_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING); } return extension->repeated_string_value->Add(); } @@ -545,7 +576,7 @@ const MessageLite& ExtensionSet::GetMessage( // Not present. Return the default value. return default_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); if (extension->is_lazy) { return extension->lazymessage_value->GetMessage(default_value); } else { @@ -572,7 +603,7 @@ MessageLite* ExtensionSet::MutableMessage(int number, FieldType type, extension->is_cleared = false; return extension->message_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); extension->is_cleared = false; if (extension->is_lazy) { return extension->lazymessage_value->MutableMessage(prototype); @@ -594,7 +625,7 @@ void ExtensionSet::SetAllocatedMessage(int number, FieldType type, ClearExtension(number); return; } - ::google::protobuf::Arena* message_arena = message->GetArena(); + Arena* message_arena = message->GetArena(); Extension* extension; if (MaybeNewExtension(number, descriptor, &extension)) { extension->type = type; @@ -611,7 +642,7 @@ void ExtensionSet::SetAllocatedMessage(int number, FieldType type, extension->message_value->CheckTypeAndMergeFrom(*message); } } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); if (extension->is_lazy) { extension->lazymessage_value->SetAllocatedMessage(message); } else { @@ -647,7 +678,7 @@ void ExtensionSet::UnsafeArenaSetAllocatedMessage( extension->is_lazy = false; extension->message_value = message; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); if (extension->is_lazy) { extension->lazymessage_value->UnsafeArenaSetAllocatedMessage(message); } else { @@ -667,7 +698,7 @@ MessageLite* ExtensionSet::ReleaseMessage(int number, // Not present. Return NULL. return NULL; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); MessageLite* ret = NULL; if (extension->is_lazy) { ret = extension->lazymessage_value->ReleaseMessage(prototype); @@ -696,7 +727,7 @@ MessageLite* ExtensionSet::UnsafeArenaReleaseMessage( // Not present. Return NULL. return NULL; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); MessageLite* ret = NULL; if (extension->is_lazy) { ret = extension->lazymessage_value->UnsafeArenaReleaseMessage(prototype); @@ -715,18 +746,18 @@ MessageLite* ExtensionSet::UnsafeArenaReleaseMessage( // MessageLite* ExtensionSet::ReleaseMessage(const FieldDescriptor* descriptor, // MessageFactory* factory); -const MessageLite& ExtensionSet::GetRepeatedMessage( - int number, int index) const { +const MessageLite& ExtensionSet::GetRepeatedMessage(int number, + int index) const { const Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE); return extension->repeated_message_value->Get(index); } MessageLite* ExtensionSet::MutableRepeatedMessage(int number, int index) { Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE); return extension->repeated_message_value->Mutable(index); } @@ -739,17 +770,16 @@ MessageLite* ExtensionSet::AddMessage(int number, FieldType type, GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_MESSAGE); extension->is_repeated = true; extension->repeated_message_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE); } // RepeatedPtrField does not know how to Add() since it cannot // allocate an abstract object, so we have to be tricky. - MessageLite* result = - reinterpret_cast<::google::protobuf::internal::RepeatedPtrFieldBase*>( - extension->repeated_message_value) - ->AddFromCleared >(); + MessageLite* result = reinterpret_cast( + extension->repeated_message_value) + ->AddFromCleared>(); if (result == NULL) { result = prototype.New(arena_); extension->repeated_message_value->AddAllocated(result); @@ -769,7 +799,7 @@ void ExtensionSet::RemoveLast(int number) { GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; GOOGLE_DCHECK(extension->is_repeated); - switch(cpp_type(extension->type)) { + switch (cpp_type(extension->type)) { case WireFormatLite::CPPTYPE_INT32: extension->repeated_int32_value->RemoveLast(); break; @@ -816,7 +846,7 @@ void ExtensionSet::SwapElements(int number, int index1, int index2) { GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; GOOGLE_DCHECK(extension->is_repeated); - switch(cpp_type(extension->type)) { + switch (cpp_type(extension->type)) { case WireFormatLite::CPPTYPE_INT32: extension->repeated_int32_value->SwapElements(index1, index2); break; @@ -879,8 +909,8 @@ size_t SizeOfUnion(ItX it_xs, ItX end_xs, ItY it_ys, ItY end_ys) { } // namespace void ExtensionSet::MergeFrom(const ExtensionSet& other) { - if (GOOGLE_PREDICT_TRUE(!is_large())) { - if (GOOGLE_PREDICT_TRUE(!other.is_large())) { + if (PROTOBUF_PREDICT_TRUE(!is_large())) { + if (PROTOBUF_PREDICT_TRUE(!other.is_large())) { GrowCapacity(SizeOfUnion(flat_begin(), flat_end(), other.flat_begin(), other.flat_end())); } else { @@ -895,11 +925,11 @@ void ExtensionSet::MergeFrom(const ExtensionSet& other) { } void ExtensionSet::InternalExtensionMergeFrom( - int number, const Extension& other_extension) { + int number, const Extension& other_extension) { if (other_extension.is_repeated) { Extension* extension; - bool is_new = MaybeNewExtension(number, other_extension.descriptor, - &extension); + bool is_new = + MaybeNewExtension(number, other_extension.descriptor, &extension); if (is_new) { // Extension did not already exist in set. extension->type = other_extension.type; @@ -912,31 +942,31 @@ void ExtensionSet::InternalExtensionMergeFrom( } switch (cpp_type(other_extension.type)) { -#define HANDLE_TYPE(UPPERCASE, LOWERCASE, REPEATED_TYPE) \ - case WireFormatLite::CPPTYPE_##UPPERCASE: \ - if (is_new) { \ - extension->repeated_##LOWERCASE##_value = \ - Arena::CreateMessage(arena_); \ - } \ - extension->repeated_##LOWERCASE##_value->MergeFrom( \ - *other_extension.repeated_##LOWERCASE##_value); \ - break; - - HANDLE_TYPE( INT32, int32, RepeatedField < int32>); - HANDLE_TYPE( INT64, int64, RepeatedField < int64>); - HANDLE_TYPE( UINT32, uint32, RepeatedField < uint32>); - HANDLE_TYPE( UINT64, uint64, RepeatedField < uint64>); - HANDLE_TYPE( FLOAT, float, RepeatedField < float>); - HANDLE_TYPE( DOUBLE, double, RepeatedField < double>); - HANDLE_TYPE( BOOL, bool, RepeatedField < bool>); - HANDLE_TYPE( ENUM, enum, RepeatedField < int>); - HANDLE_TYPE( STRING, string, RepeatedPtrField< string>); +#define HANDLE_TYPE(UPPERCASE, LOWERCASE, REPEATED_TYPE) \ + case WireFormatLite::CPPTYPE_##UPPERCASE: \ + if (is_new) { \ + extension->repeated_##LOWERCASE##_value = \ + Arena::CreateMessage(arena_); \ + } \ + extension->repeated_##LOWERCASE##_value->MergeFrom( \ + *other_extension.repeated_##LOWERCASE##_value); \ + break; + + HANDLE_TYPE(INT32, int32, RepeatedField); + HANDLE_TYPE(INT64, int64, RepeatedField); + HANDLE_TYPE(UINT32, uint32, RepeatedField); + HANDLE_TYPE(UINT64, uint64, RepeatedField); + HANDLE_TYPE(FLOAT, float, RepeatedField); + HANDLE_TYPE(DOUBLE, double, RepeatedField); + HANDLE_TYPE(BOOL, bool, RepeatedField); + HANDLE_TYPE(ENUM, enum, RepeatedField); + HANDLE_TYPE(STRING, string, RepeatedPtrField); #undef HANDLE_TYPE case WireFormatLite::CPPTYPE_MESSAGE: if (is_new) { extension->repeated_message_value = - Arena::CreateMessage >(arena_); + Arena::CreateMessage>(arena_); } // We can't call RepeatedPtrField::MergeFrom() because // it would attempt to allocate new objects. @@ -945,9 +975,9 @@ void ExtensionSet::InternalExtensionMergeFrom( for (int i = 0; i < other_repeated_message->size(); i++) { const MessageLite& other_message = other_repeated_message->Get(i); MessageLite* target = - reinterpret_cast<::google::protobuf::internal::RepeatedPtrFieldBase*>( + reinterpret_cast( extension->repeated_message_value) - ->AddFromCleared >(); + ->AddFromCleared>(); if (target == NULL) { target = other_message.New(arena_); extension->repeated_message_value->AddAllocated(target); @@ -959,32 +989,30 @@ void ExtensionSet::InternalExtensionMergeFrom( } else { if (!other_extension.is_cleared) { switch (cpp_type(other_extension.type)) { -#define HANDLE_TYPE(UPPERCASE, LOWERCASE, CAMELCASE) \ - case WireFormatLite::CPPTYPE_##UPPERCASE: \ - Set##CAMELCASE(number, other_extension.type, \ - other_extension.LOWERCASE##_value, \ - other_extension.descriptor); \ - break; - - HANDLE_TYPE( INT32, int32, Int32); - HANDLE_TYPE( INT64, int64, Int64); +#define HANDLE_TYPE(UPPERCASE, LOWERCASE, CAMELCASE) \ + case WireFormatLite::CPPTYPE_##UPPERCASE: \ + Set##CAMELCASE(number, other_extension.type, \ + other_extension.LOWERCASE##_value, \ + other_extension.descriptor); \ + break; + + HANDLE_TYPE(INT32, int32, Int32); + HANDLE_TYPE(INT64, int64, Int64); HANDLE_TYPE(UINT32, uint32, UInt32); HANDLE_TYPE(UINT64, uint64, UInt64); - HANDLE_TYPE( FLOAT, float, Float); + HANDLE_TYPE(FLOAT, float, Float); HANDLE_TYPE(DOUBLE, double, Double); - HANDLE_TYPE( BOOL, bool, Bool); - HANDLE_TYPE( ENUM, enum, Enum); + HANDLE_TYPE(BOOL, bool, Bool); + HANDLE_TYPE(ENUM, enum, Enum); #undef HANDLE_TYPE case WireFormatLite::CPPTYPE_STRING: - SetString(number, other_extension.type, - *other_extension.string_value, + SetString(number, other_extension.type, *other_extension.string_value, other_extension.descriptor); break; case WireFormatLite::CPPTYPE_MESSAGE: { Extension* extension; - bool is_new = MaybeNewExtension(number, - other_extension.descriptor, - &extension); + bool is_new = + MaybeNewExtension(number, other_extension.descriptor, &extension); if (is_new) { extension->type = other_extension.type; extension->is_packed = other_extension.is_packed; @@ -1004,7 +1032,7 @@ void ExtensionSet::InternalExtensionMergeFrom( } } else { GOOGLE_DCHECK_EQ(extension->type, other_extension.type); - GOOGLE_DCHECK_EQ(extension->is_packed,other_extension.is_packed); + GOOGLE_DCHECK_EQ(extension->is_packed, other_extension.is_packed); GOOGLE_DCHECK(!extension->is_repeated); if (other_extension.is_lazy) { if (extension->is_lazy) { @@ -1017,9 +1045,9 @@ void ExtensionSet::InternalExtensionMergeFrom( } } else { if (extension->is_lazy) { - extension->lazymessage_value->MutableMessage( - *other_extension.message_value)->CheckTypeAndMergeFrom( - *other_extension.message_value); + extension->lazymessage_value + ->MutableMessage(*other_extension.message_value) + ->CheckTypeAndMergeFrom(*other_extension.message_value); } else { extension->message_value->CheckTypeAndMergeFrom( *other_extension.message_value); @@ -1035,7 +1063,7 @@ void ExtensionSet::InternalExtensionMergeFrom( } void ExtensionSet::Swap(ExtensionSet* x) { - if (GetArenaNoVirtual() == x->GetArenaNoVirtual()) { + if (GetArena() == x->GetArena()) { using std::swap; swap(flat_capacity_, x->flat_capacity_); swap(flat_size_, x->flat_size_); @@ -1053,8 +1081,7 @@ void ExtensionSet::Swap(ExtensionSet* x) { } } -void ExtensionSet::SwapExtension(ExtensionSet* other, - int number) { +void ExtensionSet::SwapExtension(ExtensionSet* other, int number) { if (this == other) return; Extension* this_ext = FindOrNull(number); Extension* other_ext = other->FindOrNull(number); @@ -1064,7 +1091,7 @@ void ExtensionSet::SwapExtension(ExtensionSet* other, } if (this_ext != NULL && other_ext != NULL) { - if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + if (GetArena() == other->GetArena()) { using std::swap; swap(*this_ext, *other_ext); } else { @@ -1085,7 +1112,7 @@ void ExtensionSet::SwapExtension(ExtensionSet* other, } if (this_ext == NULL) { - if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + if (GetArena() == other->GetArena()) { *Insert(number).first = *other_ext; } else { InternalExtensionMergeFrom(number, *other_ext); @@ -1095,7 +1122,7 @@ void ExtensionSet::SwapExtension(ExtensionSet* other, } if (other_ext == NULL) { - if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + if (GetArena() == other->GetArena()) { *other->Insert(number).first = *this_ext; } else { other->InternalExtensionMergeFrom(number, *this_ext); @@ -1108,7 +1135,7 @@ void ExtensionSet::SwapExtension(ExtensionSet* other, bool ExtensionSet::IsInitialized() const { // Extensions are never required. However, we need to check that all // embedded messages are initialized. - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { for (const auto& kv : *map_.large) { if (!kv.second.IsInitialized()) return false; } @@ -1120,9 +1147,11 @@ bool ExtensionSet::IsInitialized() const { return true; } -bool ExtensionSet::FindExtensionInfoFromTag( - uint32 tag, ExtensionFinder* extension_finder, int* field_number, - ExtensionInfo* extension, bool* was_packed_on_wire) { +bool ExtensionSet::FindExtensionInfoFromTag(uint32 tag, + ExtensionFinder* extension_finder, + int* field_number, + ExtensionInfo* extension, + bool* was_packed_on_wire) { *field_number = WireFormatLite::GetTagFieldNumber(tag); WireFormatLite::WireType wire_type = WireFormatLite::GetTagWireType(tag); return FindExtensionInfoFromFieldNumber(wire_type, *field_number, @@ -1158,19 +1187,44 @@ bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input, int number; bool was_packed_on_wire; ExtensionInfo extension; - if (!FindExtensionInfoFromTag( - tag, extension_finder, &number, &extension, &was_packed_on_wire)) { + if (!FindExtensionInfoFromTag(tag, extension_finder, &number, &extension, + &was_packed_on_wire)) { return field_skipper->SkipField(input, tag); } else { - return ParseFieldWithExtensionInfo( - number, was_packed_on_wire, extension, input, field_skipper); + return ParseFieldWithExtensionInfo(number, was_packed_on_wire, extension, + input, field_skipper); + } +} + +const char* ExtensionSet::ParseField(uint64 tag, const char* ptr, + const MessageLite* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx) { + GeneratedExtensionFinder finder(containing_type); + int number = tag >> 3; + bool was_packed_on_wire; + ExtensionInfo extension; + if (!FindExtensionInfoFromFieldNumber(tag & 7, number, &finder, &extension, + &was_packed_on_wire)) { + return UnknownFieldParse( + tag, metadata->mutable_unknown_fields(), ptr, ctx); } + return ParseFieldWithExtensionInfo( + number, was_packed_on_wire, extension, metadata, ptr, ctx); } -bool ExtensionSet::ParseFieldWithExtensionInfo( - int number, bool was_packed_on_wire, const ExtensionInfo& extension, - io::CodedInputStream* input, - FieldSkipper* field_skipper) { +const char* ExtensionSet::ParseMessageSetItem( + const char* ptr, const MessageLite* containing_type, + internal::InternalMetadata* metadata, internal::ParseContext* ctx) { + return ParseMessageSetItemTmpl(ptr, containing_type, + metadata, ctx); +} + +bool ExtensionSet::ParseFieldWithExtensionInfo(int number, + bool was_packed_on_wire, + const ExtensionInfo& extension, + io::CodedInputStream* input, + FieldSkipper* field_skipper) { // Explicitly not read extension.is_packed, instead check whether the field // was encoded in packed form on the wire. if (was_packed_on_wire) { @@ -1179,39 +1233,40 @@ bool ExtensionSet::ParseFieldWithExtensionInfo( io::CodedInputStream::Limit limit = input->PushLimit(size); switch (extension.type) { -#define HANDLE_TYPE(UPPERCASE, CPP_CAMELCASE, CPP_LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - while (input->BytesUntilLimit() > 0) { \ - CPP_LOWERCASE value; \ - if (!WireFormatLite::ReadPrimitive< \ - CPP_LOWERCASE, WireFormatLite::TYPE_##UPPERCASE>( \ - input, &value)) return false; \ - Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, \ - extension.is_packed, value, \ - extension.descriptor); \ - } \ - break - - HANDLE_TYPE( INT32, Int32, int32); - HANDLE_TYPE( INT64, Int64, int64); - HANDLE_TYPE( UINT32, UInt32, uint32); - HANDLE_TYPE( UINT64, UInt64, uint64); - HANDLE_TYPE( SINT32, Int32, int32); - HANDLE_TYPE( SINT64, Int64, int64); - HANDLE_TYPE( FIXED32, UInt32, uint32); - HANDLE_TYPE( FIXED64, UInt64, uint64); - HANDLE_TYPE(SFIXED32, Int32, int32); - HANDLE_TYPE(SFIXED64, Int64, int64); - HANDLE_TYPE( FLOAT, Float, float); - HANDLE_TYPE( DOUBLE, Double, double); - HANDLE_TYPE( BOOL, Bool, bool); +#define HANDLE_TYPE(UPPERCASE, CPP_CAMELCASE, CPP_LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + while (input->BytesUntilLimit() > 0) { \ + CPP_LOWERCASE value; \ + if (!WireFormatLite::ReadPrimitive( \ + input, &value)) \ + return false; \ + Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, \ + extension.is_packed, value, extension.descriptor); \ + } \ + break + + HANDLE_TYPE(INT32, Int32, int32); + HANDLE_TYPE(INT64, Int64, int64); + HANDLE_TYPE(UINT32, UInt32, uint32); + HANDLE_TYPE(UINT64, UInt64, uint64); + HANDLE_TYPE(SINT32, Int32, int32); + HANDLE_TYPE(SINT64, Int64, int64); + HANDLE_TYPE(FIXED32, UInt32, uint32); + HANDLE_TYPE(FIXED64, UInt64, uint64); + HANDLE_TYPE(SFIXED32, Int32, int32); + HANDLE_TYPE(SFIXED64, Int64, int64); + HANDLE_TYPE(FLOAT, Float, float); + HANDLE_TYPE(DOUBLE, Double, double); + HANDLE_TYPE(BOOL, Bool, bool); #undef HANDLE_TYPE case WireFormatLite::TYPE_ENUM: while (input->BytesUntilLimit() > 0) { int value; if (!WireFormatLite::ReadPrimitive( - input, &value)) return false; + input, &value)) + return false; if (extension.enum_validity_check.func( extension.enum_validity_check.arg, value)) { AddEnum(number, WireFormatLite::TYPE_ENUM, extension.is_packed, @@ -1234,41 +1289,42 @@ bool ExtensionSet::ParseFieldWithExtensionInfo( input->PopLimit(limit); } else { switch (extension.type) { -#define HANDLE_TYPE(UPPERCASE, CPP_CAMELCASE, CPP_LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: { \ - CPP_LOWERCASE value; \ - if (!WireFormatLite::ReadPrimitive< \ - CPP_LOWERCASE, WireFormatLite::TYPE_##UPPERCASE>( \ - input, &value)) return false; \ - if (extension.is_repeated) { \ - Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, \ - extension.is_packed, value, \ - extension.descriptor); \ - } else { \ - Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value, \ - extension.descriptor); \ - } \ - } break - - HANDLE_TYPE( INT32, Int32, int32); - HANDLE_TYPE( INT64, Int64, int64); - HANDLE_TYPE( UINT32, UInt32, uint32); - HANDLE_TYPE( UINT64, UInt64, uint64); - HANDLE_TYPE( SINT32, Int32, int32); - HANDLE_TYPE( SINT64, Int64, int64); - HANDLE_TYPE( FIXED32, UInt32, uint32); - HANDLE_TYPE( FIXED64, UInt64, uint64); - HANDLE_TYPE(SFIXED32, Int32, int32); - HANDLE_TYPE(SFIXED64, Int64, int64); - HANDLE_TYPE( FLOAT, Float, float); - HANDLE_TYPE( DOUBLE, Double, double); - HANDLE_TYPE( BOOL, Bool, bool); +#define HANDLE_TYPE(UPPERCASE, CPP_CAMELCASE, CPP_LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: { \ + CPP_LOWERCASE value; \ + if (!WireFormatLite::ReadPrimitive( \ + input, &value)) \ + return false; \ + if (extension.is_repeated) { \ + Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, \ + extension.is_packed, value, extension.descriptor); \ + } else { \ + Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value, \ + extension.descriptor); \ + } \ + } break + + HANDLE_TYPE(INT32, Int32, int32); + HANDLE_TYPE(INT64, Int64, int64); + HANDLE_TYPE(UINT32, UInt32, uint32); + HANDLE_TYPE(UINT64, UInt64, uint64); + HANDLE_TYPE(SINT32, Int32, int32); + HANDLE_TYPE(SINT64, Int64, int64); + HANDLE_TYPE(FIXED32, UInt32, uint32); + HANDLE_TYPE(FIXED64, UInt64, uint64); + HANDLE_TYPE(SFIXED32, Int32, int32); + HANDLE_TYPE(SFIXED64, Int64, int64); + HANDLE_TYPE(FLOAT, Float, float); + HANDLE_TYPE(DOUBLE, Double, double); + HANDLE_TYPE(BOOL, Bool, bool); #undef HANDLE_TYPE case WireFormatLite::TYPE_ENUM: { int value; if (!WireFormatLite::ReadPrimitive( - input, &value)) return false; + input, &value)) + return false; if (!extension.enum_validity_check.func( extension.enum_validity_check.arg, value)) { @@ -1284,40 +1340,50 @@ bool ExtensionSet::ParseFieldWithExtensionInfo( break; } - case WireFormatLite::TYPE_STRING: { - string* value = extension.is_repeated ? - AddString(number, WireFormatLite::TYPE_STRING, extension.descriptor) : - MutableString(number, WireFormatLite::TYPE_STRING, - extension.descriptor); + case WireFormatLite::TYPE_STRING: { + std::string* value = + extension.is_repeated + ? AddString(number, WireFormatLite::TYPE_STRING, + extension.descriptor) + : MutableString(number, WireFormatLite::TYPE_STRING, + extension.descriptor); if (!WireFormatLite::ReadString(input, value)) return false; break; } - case WireFormatLite::TYPE_BYTES: { - string* value = extension.is_repeated ? - AddString(number, WireFormatLite::TYPE_BYTES, extension.descriptor) : - MutableString(number, WireFormatLite::TYPE_BYTES, - extension.descriptor); + case WireFormatLite::TYPE_BYTES: { + std::string* value = + extension.is_repeated + ? AddString(number, WireFormatLite::TYPE_BYTES, + extension.descriptor) + : MutableString(number, WireFormatLite::TYPE_BYTES, + extension.descriptor); if (!WireFormatLite::ReadBytes(input, value)) return false; break; } case WireFormatLite::TYPE_GROUP: { - MessageLite* value = extension.is_repeated ? - AddMessage(number, WireFormatLite::TYPE_GROUP, - *extension.message_prototype, extension.descriptor) : - MutableMessage(number, WireFormatLite::TYPE_GROUP, - *extension.message_prototype, extension.descriptor); + MessageLite* value = + extension.is_repeated + ? AddMessage(number, WireFormatLite::TYPE_GROUP, + *extension.message_info.prototype, + extension.descriptor) + : MutableMessage(number, WireFormatLite::TYPE_GROUP, + *extension.message_info.prototype, + extension.descriptor); if (!WireFormatLite::ReadGroup(number, input, value)) return false; break; } case WireFormatLite::TYPE_MESSAGE: { - MessageLite* value = extension.is_repeated ? - AddMessage(number, WireFormatLite::TYPE_MESSAGE, - *extension.message_prototype, extension.descriptor) : - MutableMessage(number, WireFormatLite::TYPE_MESSAGE, - *extension.message_prototype, extension.descriptor); + MessageLite* value = + extension.is_repeated + ? AddMessage(number, WireFormatLite::TYPE_MESSAGE, + *extension.message_info.prototype, + extension.descriptor) + : MutableMessage(number, WireFormatLite::TYPE_MESSAGE, + *extension.message_info.prototype, + extension.descriptor); if (!WireFormatLite::ReadMessage(input, value)) return false; break; } @@ -1342,33 +1408,90 @@ bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input, return ParseField(tag, input, &finder, &skipper); } -// Defined in extension_set_heavy.cc. -// bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input, -// const MessageLite* containing_type, -// UnknownFieldSet* unknown_fields) +bool ExtensionSet::ParseMessageSetLite(io::CodedInputStream* input, + ExtensionFinder* extension_finder, + FieldSkipper* field_skipper) { + while (true) { + const uint32 tag = input->ReadTag(); + switch (tag) { + case 0: + return true; + case WireFormatLite::kMessageSetItemStartTag: + if (!ParseMessageSetItemLite(input, extension_finder, field_skipper)) { + return false; + } + break; + default: + if (!ParseField(tag, input, extension_finder, field_skipper)) { + return false; + } + break; + } + } +} -// Defined in extension_set_heavy.cc. -// bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input, -// const MessageLite* containing_type, -// UnknownFieldSet* unknown_fields); - -void ExtensionSet::SerializeWithCachedSizes( - int start_field_number, int end_field_number, - io::CodedOutputStream* output) const { - if (GOOGLE_PREDICT_FALSE(is_large())) { +bool ExtensionSet::ParseMessageSetItemLite(io::CodedInputStream* input, + ExtensionFinder* extension_finder, + FieldSkipper* field_skipper) { + struct MSLite { + bool ParseField(int type_id, io::CodedInputStream* input) { + return me->ParseField( + WireFormatLite::WIRETYPE_LENGTH_DELIMITED + 8 * type_id, input, + extension_finder, field_skipper); + } + + bool SkipField(uint32 tag, io::CodedInputStream* input) { + return field_skipper->SkipField(input, tag); + } + + ExtensionSet* me; + ExtensionFinder* extension_finder; + FieldSkipper* field_skipper; + }; + + return ParseMessageSetItemImpl(input, + MSLite{this, extension_finder, field_skipper}); +} + +bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input, + const MessageLite* containing_type, + std::string* unknown_fields) { + io::StringOutputStream zcis(unknown_fields); + io::CodedOutputStream output(&zcis); + CodedOutputStreamFieldSkipper skipper(&output); + GeneratedExtensionFinder finder(containing_type); + return ParseMessageSetLite(input, &finder, &skipper); +} + +uint8* ExtensionSet::_InternalSerialize(int start_field_number, + int end_field_number, uint8* target, + io::EpsCopyOutputStream* stream) const { + if (PROTOBUF_PREDICT_FALSE(is_large())) { const auto& end = map_.large->end(); for (auto it = map_.large->lower_bound(start_field_number); it != end && it->first < end_field_number; ++it) { - it->second.SerializeFieldWithCachedSizes(it->first, output); + target = it->second.InternalSerializeFieldWithCachedSizesToArray( + it->first, target, stream); } - return; + return target; } const KeyValue* end = flat_end(); for (const KeyValue* it = std::lower_bound( flat_begin(), end, start_field_number, KeyValue::FirstComparator()); it != end && it->first < end_field_number; ++it) { - it->second.SerializeFieldWithCachedSizes(it->first, output); + target = it->second.InternalSerializeFieldWithCachedSizesToArray( + it->first, target, stream); } + return target; +} + +uint8* ExtensionSet::InternalSerializeMessageSetWithCachedSizesToArray( + uint8* target, io::EpsCopyOutputStream* stream) const { + ForEach([&target, stream](int number, const Extension& ext) { + target = ext.InternalSerializeMessageSetItemWithCachedSizesToArray( + number, target, stream); + }); + return target; } size_t ExtensionSet::ByteSize() const { @@ -1397,20 +1520,20 @@ bool ExtensionSet::MaybeNewExtension(int number, void ExtensionSet::Extension::Clear() { if (is_repeated) { switch (cpp_type(type)) { -#define HANDLE_TYPE(UPPERCASE, LOWERCASE) \ - case WireFormatLite::CPPTYPE_##UPPERCASE: \ - repeated_##LOWERCASE##_value->Clear(); \ - break - - HANDLE_TYPE( INT32, int32); - HANDLE_TYPE( INT64, int64); - HANDLE_TYPE( UINT32, uint32); - HANDLE_TYPE( UINT64, uint64); - HANDLE_TYPE( FLOAT, float); - HANDLE_TYPE( DOUBLE, double); - HANDLE_TYPE( BOOL, bool); - HANDLE_TYPE( ENUM, enum); - HANDLE_TYPE( STRING, string); +#define HANDLE_TYPE(UPPERCASE, LOWERCASE) \ + case WireFormatLite::CPPTYPE_##UPPERCASE: \ + repeated_##LOWERCASE##_value->Clear(); \ + break + + HANDLE_TYPE(INT32, int32); + HANDLE_TYPE(INT64, int64); + HANDLE_TYPE(UINT32, uint32); + HANDLE_TYPE(UINT64, uint64); + HANDLE_TYPE(FLOAT, float); + HANDLE_TYPE(DOUBLE, double); + HANDLE_TYPE(BOOL, bool); + HANDLE_TYPE(ENUM, enum); + HANDLE_TYPE(STRING, string); HANDLE_TYPE(MESSAGE, message); #undef HANDLE_TYPE } @@ -1439,152 +1562,42 @@ void ExtensionSet::Extension::Clear() { } } -void ExtensionSet::Extension::SerializeFieldWithCachedSizes( - int number, - io::CodedOutputStream* output) const { - if (is_repeated) { - if (is_packed) { - if (cached_size == 0) return; - - WireFormatLite::WriteTag(number, - WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output); - output->WriteVarint32(cached_size); - - switch (real_type(type)) { -#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ - WireFormatLite::Write##CAMELCASE##NoTag( \ - repeated_##LOWERCASE##_value->Get(i), output); \ - } \ - break - - HANDLE_TYPE( INT32, Int32, int32); - HANDLE_TYPE( INT64, Int64, int64); - HANDLE_TYPE( UINT32, UInt32, uint32); - HANDLE_TYPE( UINT64, UInt64, uint64); - HANDLE_TYPE( SINT32, SInt32, int32); - HANDLE_TYPE( SINT64, SInt64, int64); - HANDLE_TYPE( FIXED32, Fixed32, uint32); - HANDLE_TYPE( FIXED64, Fixed64, uint64); - HANDLE_TYPE(SFIXED32, SFixed32, int32); - HANDLE_TYPE(SFIXED64, SFixed64, int64); - HANDLE_TYPE( FLOAT, Float, float); - HANDLE_TYPE( DOUBLE, Double, double); - HANDLE_TYPE( BOOL, Bool, bool); - HANDLE_TYPE( ENUM, Enum, enum); -#undef HANDLE_TYPE - - case WireFormatLite::TYPE_STRING: - case WireFormatLite::TYPE_BYTES: - case WireFormatLite::TYPE_GROUP: - case WireFormatLite::TYPE_MESSAGE: - GOOGLE_LOG(FATAL) << "Non-primitive types can't be packed."; - break; - } - } else { - switch (real_type(type)) { -#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ - WireFormatLite::Write##CAMELCASE(number, \ - repeated_##LOWERCASE##_value->Get(i), output); \ - } \ - break - - HANDLE_TYPE( INT32, Int32, int32); - HANDLE_TYPE( INT64, Int64, int64); - HANDLE_TYPE( UINT32, UInt32, uint32); - HANDLE_TYPE( UINT64, UInt64, uint64); - HANDLE_TYPE( SINT32, SInt32, int32); - HANDLE_TYPE( SINT64, SInt64, int64); - HANDLE_TYPE( FIXED32, Fixed32, uint32); - HANDLE_TYPE( FIXED64, Fixed64, uint64); - HANDLE_TYPE(SFIXED32, SFixed32, int32); - HANDLE_TYPE(SFIXED64, SFixed64, int64); - HANDLE_TYPE( FLOAT, Float, float); - HANDLE_TYPE( DOUBLE, Double, double); - HANDLE_TYPE( BOOL, Bool, bool); - HANDLE_TYPE( STRING, String, string); - HANDLE_TYPE( BYTES, Bytes, string); - HANDLE_TYPE( ENUM, Enum, enum); - HANDLE_TYPE( GROUP, Group, message); - HANDLE_TYPE( MESSAGE, Message, message); -#undef HANDLE_TYPE - } - } - } else if (!is_cleared) { - switch (real_type(type)) { -#define HANDLE_TYPE(UPPERCASE, CAMELCASE, VALUE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - WireFormatLite::Write##CAMELCASE(number, VALUE, output); \ - break - - HANDLE_TYPE( INT32, Int32, int32_value); - HANDLE_TYPE( INT64, Int64, int64_value); - HANDLE_TYPE( UINT32, UInt32, uint32_value); - HANDLE_TYPE( UINT64, UInt64, uint64_value); - HANDLE_TYPE( SINT32, SInt32, int32_value); - HANDLE_TYPE( SINT64, SInt64, int64_value); - HANDLE_TYPE( FIXED32, Fixed32, uint32_value); - HANDLE_TYPE( FIXED64, Fixed64, uint64_value); - HANDLE_TYPE(SFIXED32, SFixed32, int32_value); - HANDLE_TYPE(SFIXED64, SFixed64, int64_value); - HANDLE_TYPE( FLOAT, Float, float_value); - HANDLE_TYPE( DOUBLE, Double, double_value); - HANDLE_TYPE( BOOL, Bool, bool_value); - HANDLE_TYPE( STRING, String, *string_value); - HANDLE_TYPE( BYTES, Bytes, *string_value); - HANDLE_TYPE( ENUM, Enum, enum_value); - HANDLE_TYPE( GROUP, Group, *message_value); -#undef HANDLE_TYPE - case WireFormatLite::TYPE_MESSAGE: - if (is_lazy) { - lazymessage_value->WriteMessage(number, output); - } else { - WireFormatLite::WriteMessage(number, *message_value, output); - } - break; - } - } -} - size_t ExtensionSet::Extension::ByteSize(int number) const { size_t result = 0; if (is_repeated) { if (is_packed) { switch (real_type(type)) { -#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ - result += WireFormatLite::CAMELCASE##Size( \ - repeated_##LOWERCASE##_value->Get(i)); \ - } \ - break - - HANDLE_TYPE( INT32, Int32, int32); - HANDLE_TYPE( INT64, Int64, int64); - HANDLE_TYPE( UINT32, UInt32, uint32); - HANDLE_TYPE( UINT64, UInt64, uint64); - HANDLE_TYPE( SINT32, SInt32, int32); - HANDLE_TYPE( SINT64, SInt64, int64); - HANDLE_TYPE( ENUM, Enum, enum); +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ + result += WireFormatLite::CAMELCASE##Size( \ + repeated_##LOWERCASE##_value->Get(i)); \ + } \ + break + + HANDLE_TYPE(INT32, Int32, int32); + HANDLE_TYPE(INT64, Int64, int64); + HANDLE_TYPE(UINT32, UInt32, uint32); + HANDLE_TYPE(UINT64, UInt64, uint64); + HANDLE_TYPE(SINT32, SInt32, int32); + HANDLE_TYPE(SINT64, SInt64, int64); + HANDLE_TYPE(ENUM, Enum, enum); #undef HANDLE_TYPE // Stuff with fixed size. -#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - result += WireFormatLite::k##CAMELCASE##Size * \ - FromIntSize(repeated_##LOWERCASE##_value->size()); \ - break - HANDLE_TYPE( FIXED32, Fixed32, uint32); - HANDLE_TYPE( FIXED64, Fixed64, uint64); - HANDLE_TYPE(SFIXED32, SFixed32, int32); - HANDLE_TYPE(SFIXED64, SFixed64, int64); - HANDLE_TYPE( FLOAT, Float, float); - HANDLE_TYPE( DOUBLE, Double, double); - HANDLE_TYPE( BOOL, Bool, bool); +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + result += WireFormatLite::k##CAMELCASE##Size * \ + FromIntSize(repeated_##LOWERCASE##_value->size()); \ + break + HANDLE_TYPE(FIXED32, Fixed32, uint32); + HANDLE_TYPE(FIXED64, Fixed64, uint64); + HANDLE_TYPE(SFIXED32, SFixed32, int32); + HANDLE_TYPE(SFIXED64, SFixed64, int64); + HANDLE_TYPE(FLOAT, Float, float); + HANDLE_TYPE(DOUBLE, Double, double); + HANDLE_TYPE(BOOL, Bool, bool); #undef HANDLE_TYPE case WireFormatLite::TYPE_STRING: @@ -1598,75 +1611,73 @@ size_t ExtensionSet::Extension::ByteSize(int number) const { cached_size = ToCachedSize(result); if (result > 0) { result += io::CodedOutputStream::VarintSize32(result); - result += io::CodedOutputStream::VarintSize32( - WireFormatLite::MakeTag(number, - WireFormatLite::WIRETYPE_LENGTH_DELIMITED)); + result += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag( + number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED)); } } else { size_t tag_size = WireFormatLite::TagSize(number, real_type(type)); switch (real_type(type)) { #define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - result += tag_size * \ - FromIntSize(repeated_##LOWERCASE##_value->size()); \ - for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ - result += WireFormatLite::CAMELCASE##Size( \ - repeated_##LOWERCASE##_value->Get(i)); \ - } \ - break - - HANDLE_TYPE( INT32, Int32, int32); - HANDLE_TYPE( INT64, Int64, int64); - HANDLE_TYPE( UINT32, UInt32, uint32); - HANDLE_TYPE( UINT64, UInt64, uint64); - HANDLE_TYPE( SINT32, SInt32, int32); - HANDLE_TYPE( SINT64, SInt64, int64); - HANDLE_TYPE( STRING, String, string); - HANDLE_TYPE( BYTES, Bytes, string); - HANDLE_TYPE( ENUM, Enum, enum); - HANDLE_TYPE( GROUP, Group, message); - HANDLE_TYPE( MESSAGE, Message, message); + case WireFormatLite::TYPE_##UPPERCASE: \ + result += tag_size * FromIntSize(repeated_##LOWERCASE##_value->size()); \ + for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ + result += WireFormatLite::CAMELCASE##Size( \ + repeated_##LOWERCASE##_value->Get(i)); \ + } \ + break + + HANDLE_TYPE(INT32, Int32, int32); + HANDLE_TYPE(INT64, Int64, int64); + HANDLE_TYPE(UINT32, UInt32, uint32); + HANDLE_TYPE(UINT64, UInt64, uint64); + HANDLE_TYPE(SINT32, SInt32, int32); + HANDLE_TYPE(SINT64, SInt64, int64); + HANDLE_TYPE(STRING, String, string); + HANDLE_TYPE(BYTES, Bytes, string); + HANDLE_TYPE(ENUM, Enum, enum); + HANDLE_TYPE(GROUP, Group, message); + HANDLE_TYPE(MESSAGE, Message, message); #undef HANDLE_TYPE // Stuff with fixed size. -#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - result += (tag_size + WireFormatLite::k##CAMELCASE##Size) * \ - FromIntSize(repeated_##LOWERCASE##_value->size()); \ - break - HANDLE_TYPE( FIXED32, Fixed32, uint32); - HANDLE_TYPE( FIXED64, Fixed64, uint64); - HANDLE_TYPE(SFIXED32, SFixed32, int32); - HANDLE_TYPE(SFIXED64, SFixed64, int64); - HANDLE_TYPE( FLOAT, Float, float); - HANDLE_TYPE( DOUBLE, Double, double); - HANDLE_TYPE( BOOL, Bool, bool); +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + result += (tag_size + WireFormatLite::k##CAMELCASE##Size) * \ + FromIntSize(repeated_##LOWERCASE##_value->size()); \ + break + HANDLE_TYPE(FIXED32, Fixed32, uint32); + HANDLE_TYPE(FIXED64, Fixed64, uint64); + HANDLE_TYPE(SFIXED32, SFixed32, int32); + HANDLE_TYPE(SFIXED64, SFixed64, int64); + HANDLE_TYPE(FLOAT, Float, float); + HANDLE_TYPE(DOUBLE, Double, double); + HANDLE_TYPE(BOOL, Bool, bool); #undef HANDLE_TYPE } } } else if (!is_cleared) { result += WireFormatLite::TagSize(number, real_type(type)); switch (real_type(type)) { -#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - result += WireFormatLite::CAMELCASE##Size(LOWERCASE); \ - break - - HANDLE_TYPE( INT32, Int32, int32_value); - HANDLE_TYPE( INT64, Int64, int64_value); - HANDLE_TYPE( UINT32, UInt32, uint32_value); - HANDLE_TYPE( UINT64, UInt64, uint64_value); - HANDLE_TYPE( SINT32, SInt32, int32_value); - HANDLE_TYPE( SINT64, SInt64, int64_value); - HANDLE_TYPE( STRING, String, *string_value); - HANDLE_TYPE( BYTES, Bytes, *string_value); - HANDLE_TYPE( ENUM, Enum, enum_value); - HANDLE_TYPE( GROUP, Group, *message_value); +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + result += WireFormatLite::CAMELCASE##Size(LOWERCASE); \ + break + + HANDLE_TYPE(INT32, Int32, int32_value); + HANDLE_TYPE(INT64, Int64, int64_value); + HANDLE_TYPE(UINT32, UInt32, uint32_value); + HANDLE_TYPE(UINT64, UInt64, uint64_value); + HANDLE_TYPE(SINT32, SInt32, int32_value); + HANDLE_TYPE(SINT64, SInt64, int64_value); + HANDLE_TYPE(STRING, String, *string_value); + HANDLE_TYPE(BYTES, Bytes, *string_value); + HANDLE_TYPE(ENUM, Enum, enum_value); + HANDLE_TYPE(GROUP, Group, *message_value); #undef HANDLE_TYPE case WireFormatLite::TYPE_MESSAGE: { if (is_lazy) { - size_t size = lazymessage_value->ByteSize(); + size_t size = lazymessage_value->ByteSizeLong(); result += io::CodedOutputStream::VarintSize32(size) + size; } else { result += WireFormatLite::MessageSize(*message_value); @@ -1675,17 +1686,17 @@ size_t ExtensionSet::Extension::ByteSize(int number) const { } // Stuff with fixed size. -#define HANDLE_TYPE(UPPERCASE, CAMELCASE) \ - case WireFormatLite::TYPE_##UPPERCASE: \ - result += WireFormatLite::k##CAMELCASE##Size; \ - break - HANDLE_TYPE( FIXED32, Fixed32); - HANDLE_TYPE( FIXED64, Fixed64); - HANDLE_TYPE(SFIXED32, SFixed32); - HANDLE_TYPE(SFIXED64, SFixed64); - HANDLE_TYPE( FLOAT, Float); - HANDLE_TYPE( DOUBLE, Double); - HANDLE_TYPE( BOOL, Bool); +#define HANDLE_TYPE(UPPERCASE, CAMELCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + result += WireFormatLite::k##CAMELCASE##Size; \ + break + HANDLE_TYPE(FIXED32, Fixed32); + HANDLE_TYPE(FIXED64, Fixed64); + HANDLE_TYPE(SFIXED32, SFixed32); + HANDLE_TYPE(SFIXED64, SFixed64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); #undef HANDLE_TYPE } } @@ -1696,19 +1707,19 @@ size_t ExtensionSet::Extension::ByteSize(int number) const { int ExtensionSet::Extension::GetSize() const { GOOGLE_DCHECK(is_repeated); switch (cpp_type(type)) { -#define HANDLE_TYPE(UPPERCASE, LOWERCASE) \ - case WireFormatLite::CPPTYPE_##UPPERCASE: \ - return repeated_##LOWERCASE##_value->size() - - HANDLE_TYPE( INT32, int32); - HANDLE_TYPE( INT64, int64); - HANDLE_TYPE( UINT32, uint32); - HANDLE_TYPE( UINT64, uint64); - HANDLE_TYPE( FLOAT, float); - HANDLE_TYPE( DOUBLE, double); - HANDLE_TYPE( BOOL, bool); - HANDLE_TYPE( ENUM, enum); - HANDLE_TYPE( STRING, string); +#define HANDLE_TYPE(UPPERCASE, LOWERCASE) \ + case WireFormatLite::CPPTYPE_##UPPERCASE: \ + return repeated_##LOWERCASE##_value->size() + + HANDLE_TYPE(INT32, int32); + HANDLE_TYPE(INT64, int64); + HANDLE_TYPE(UINT32, uint32); + HANDLE_TYPE(UINT64, uint64); + HANDLE_TYPE(FLOAT, float); + HANDLE_TYPE(DOUBLE, double); + HANDLE_TYPE(BOOL, bool); + HANDLE_TYPE(ENUM, enum); + HANDLE_TYPE(STRING, string); HANDLE_TYPE(MESSAGE, message); #undef HANDLE_TYPE } @@ -1722,20 +1733,20 @@ int ExtensionSet::Extension::GetSize() const { void ExtensionSet::Extension::Free() { if (is_repeated) { switch (cpp_type(type)) { -#define HANDLE_TYPE(UPPERCASE, LOWERCASE) \ - case WireFormatLite::CPPTYPE_##UPPERCASE: \ - delete repeated_##LOWERCASE##_value; \ - break - - HANDLE_TYPE( INT32, int32); - HANDLE_TYPE( INT64, int64); - HANDLE_TYPE( UINT32, uint32); - HANDLE_TYPE( UINT64, uint64); - HANDLE_TYPE( FLOAT, float); - HANDLE_TYPE( DOUBLE, double); - HANDLE_TYPE( BOOL, bool); - HANDLE_TYPE( ENUM, enum); - HANDLE_TYPE( STRING, string); +#define HANDLE_TYPE(UPPERCASE, LOWERCASE) \ + case WireFormatLite::CPPTYPE_##UPPERCASE: \ + delete repeated_##LOWERCASE##_value; \ + break + + HANDLE_TYPE(INT32, int32); + HANDLE_TYPE(INT64, int64); + HANDLE_TYPE(UINT32, uint32); + HANDLE_TYPE(UINT64, uint64); + HANDLE_TYPE(FLOAT, float); + HANDLE_TYPE(DOUBLE, double); + HANDLE_TYPE(BOOL, bool); + HANDLE_TYPE(ENUM, enum); + HANDLE_TYPE(STRING, string); HANDLE_TYPE(MESSAGE, message); #undef HANDLE_TYPE } @@ -1785,7 +1796,7 @@ bool ExtensionSet::Extension::IsInitialized() const { void ExtensionSet::LazyMessageExtension::UnusedKeyMethod() {} const ExtensionSet::Extension* ExtensionSet::FindOrNull(int key) const { - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { return FindOrNullInLargeMap(key); } const KeyValue* end = flat_end(); @@ -1808,7 +1819,7 @@ const ExtensionSet::Extension* ExtensionSet::FindOrNullInLargeMap( } ExtensionSet::Extension* ExtensionSet::FindOrNull(int key) { - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { return FindOrNullInLargeMap(key); } KeyValue* end = flat_end(); @@ -1830,7 +1841,7 @@ ExtensionSet::Extension* ExtensionSet::FindOrNullInLargeMap(int key) { } std::pair ExtensionSet::Insert(int key) { - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { auto maybe = map_.large->insert({key, Extension()}); return {&maybe.first->second, maybe.second}; } @@ -1852,13 +1863,15 @@ std::pair ExtensionSet::Insert(int key) { } void ExtensionSet::GrowCapacity(size_t minimum_new_capacity) { - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { return; // LargeMap does not have a "reserve" method. } if (flat_capacity_ >= minimum_new_capacity) { return; } + const auto old_flat_capacity = flat_capacity_; + do { flat_capacity_ = flat_capacity_ == 0 ? 1 : flat_capacity_ * 4; } while (flat_capacity_ < minimum_new_capacity); @@ -1867,24 +1880,26 @@ void ExtensionSet::GrowCapacity(size_t minimum_new_capacity) { const KeyValue* end = flat_end(); if (flat_capacity_ > kMaximumFlatCapacity) { // Switch to LargeMap - map_.large = ::google::protobuf::Arena::Create(arena_); + map_.large = Arena::Create(arena_); LargeMap::iterator hint = map_.large->begin(); for (const KeyValue* it = begin; it != end; ++it) { hint = map_.large->insert(hint, {it->first, it->second}); } flat_size_ = 0; } else { - map_.flat = ::google::protobuf::Arena::CreateArray(arena_, flat_capacity_); + map_.flat = Arena::CreateArray(arena_, flat_capacity_); std::copy(begin, end, map_.flat); } - if (arena_ == NULL) delete[] begin; + if (arena_ == nullptr) { + DeleteFlatMap(begin, old_flat_capacity); + } } // static constexpr uint16 ExtensionSet::kMaximumFlatCapacity; void ExtensionSet::Erase(int key) { - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { map_.large->erase(key); return; } @@ -1911,6 +1926,222 @@ RepeatedStringTypeTraits::GetDefaultRepeatedField() { return instance; } +uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray( + int number, uint8* target, io::EpsCopyOutputStream* stream) const { + if (is_repeated) { + if (is_packed) { + if (cached_size == 0) return target; + + target = stream->EnsureSpace(target); + target = WireFormatLite::WriteTagToArray( + number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, target); + target = WireFormatLite::WriteInt32NoTagToArray(cached_size, target); + + switch (real_type(type)) { +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ + target = stream->EnsureSpace(target); \ + target = WireFormatLite::Write##CAMELCASE##NoTagToArray( \ + repeated_##LOWERCASE##_value->Get(i), target); \ + } \ + break + + HANDLE_TYPE(INT32, Int32, int32); + HANDLE_TYPE(INT64, Int64, int64); + HANDLE_TYPE(UINT32, UInt32, uint32); + HANDLE_TYPE(UINT64, UInt64, uint64); + HANDLE_TYPE(SINT32, SInt32, int32); + HANDLE_TYPE(SINT64, SInt64, int64); + HANDLE_TYPE(FIXED32, Fixed32, uint32); + HANDLE_TYPE(FIXED64, Fixed64, uint64); + HANDLE_TYPE(SFIXED32, SFixed32, int32); + HANDLE_TYPE(SFIXED64, SFixed64, int64); + HANDLE_TYPE(FLOAT, Float, float); + HANDLE_TYPE(DOUBLE, Double, double); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(ENUM, Enum, enum); +#undef HANDLE_TYPE + + case WireFormatLite::TYPE_STRING: + case WireFormatLite::TYPE_BYTES: + case WireFormatLite::TYPE_GROUP: + case WireFormatLite::TYPE_MESSAGE: + GOOGLE_LOG(FATAL) << "Non-primitive types can't be packed."; + break; + } + } else { + switch (real_type(type)) { +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ + target = stream->EnsureSpace(target); \ + target = WireFormatLite::Write##CAMELCASE##ToArray( \ + number, repeated_##LOWERCASE##_value->Get(i), target); \ + } \ + break + + HANDLE_TYPE(INT32, Int32, int32); + HANDLE_TYPE(INT64, Int64, int64); + HANDLE_TYPE(UINT32, UInt32, uint32); + HANDLE_TYPE(UINT64, UInt64, uint64); + HANDLE_TYPE(SINT32, SInt32, int32); + HANDLE_TYPE(SINT64, SInt64, int64); + HANDLE_TYPE(FIXED32, Fixed32, uint32); + HANDLE_TYPE(FIXED64, Fixed64, uint64); + HANDLE_TYPE(SFIXED32, SFixed32, int32); + HANDLE_TYPE(SFIXED64, SFixed64, int64); + HANDLE_TYPE(FLOAT, Float, float); + HANDLE_TYPE(DOUBLE, Double, double); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(ENUM, Enum, enum); +#undef HANDLE_TYPE +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ + target = stream->EnsureSpace(target); \ + target = stream->WriteString( \ + number, repeated_##LOWERCASE##_value->Get(i), target); \ + } \ + break + HANDLE_TYPE(STRING, String, string); + HANDLE_TYPE(BYTES, Bytes, string); +#undef HANDLE_TYPE +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \ + target = stream->EnsureSpace(target); \ + target = WireFormatLite::InternalWrite##CAMELCASE( \ + number, repeated_##LOWERCASE##_value->Get(i), target, stream); \ + } \ + break + + HANDLE_TYPE(GROUP, Group, message); + HANDLE_TYPE(MESSAGE, Message, message); +#undef HANDLE_TYPE + } + } + } else if (!is_cleared) { + switch (real_type(type)) { +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, VALUE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + target = stream->EnsureSpace(target); \ + target = WireFormatLite::Write##CAMELCASE##ToArray(number, VALUE, target); \ + break + + HANDLE_TYPE(INT32, Int32, int32_value); + HANDLE_TYPE(INT64, Int64, int64_value); + HANDLE_TYPE(UINT32, UInt32, uint32_value); + HANDLE_TYPE(UINT64, UInt64, uint64_value); + HANDLE_TYPE(SINT32, SInt32, int32_value); + HANDLE_TYPE(SINT64, SInt64, int64_value); + HANDLE_TYPE(FIXED32, Fixed32, uint32_value); + HANDLE_TYPE(FIXED64, Fixed64, uint64_value); + HANDLE_TYPE(SFIXED32, SFixed32, int32_value); + HANDLE_TYPE(SFIXED64, SFixed64, int64_value); + HANDLE_TYPE(FLOAT, Float, float_value); + HANDLE_TYPE(DOUBLE, Double, double_value); + HANDLE_TYPE(BOOL, Bool, bool_value); + HANDLE_TYPE(ENUM, Enum, enum_value); +#undef HANDLE_TYPE +#define HANDLE_TYPE(UPPERCASE, CAMELCASE, VALUE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + target = stream->EnsureSpace(target); \ + target = stream->WriteString(number, VALUE, target); \ + break + HANDLE_TYPE(STRING, String, *string_value); + HANDLE_TYPE(BYTES, Bytes, *string_value); +#undef HANDLE_TYPE + case WireFormatLite::TYPE_GROUP: + target = stream->EnsureSpace(target); + target = WireFormatLite::InternalWriteGroup(number, *message_value, + target, stream); + break; + case WireFormatLite::TYPE_MESSAGE: + if (is_lazy) { + target = + lazymessage_value->WriteMessageToArray(number, target, stream); + } else { + target = stream->EnsureSpace(target); + target = WireFormatLite::InternalWriteMessage(number, *message_value, + target, stream); + } + break; + } + } + return target; +} + +uint8* +ExtensionSet::Extension::InternalSerializeMessageSetItemWithCachedSizesToArray( + int number, uint8* target, io::EpsCopyOutputStream* stream) const { + if (type != WireFormatLite::TYPE_MESSAGE || is_repeated) { + // Not a valid MessageSet extension, but serialize it the normal way. + GOOGLE_LOG(WARNING) << "Invalid message set extension."; + return InternalSerializeFieldWithCachedSizesToArray(number, target, stream); + } + + if (is_cleared) return target; + + target = stream->EnsureSpace(target); + // Start group. + target = io::CodedOutputStream::WriteTagToArray( + WireFormatLite::kMessageSetItemStartTag, target); + // Write type ID. + target = WireFormatLite::WriteUInt32ToArray( + WireFormatLite::kMessageSetTypeIdNumber, number, target); + // Write message. + if (is_lazy) { + target = lazymessage_value->WriteMessageToArray( + WireFormatLite::kMessageSetMessageNumber, target, stream); + } else { + target = WireFormatLite::InternalWriteMessage( + WireFormatLite::kMessageSetMessageNumber, *message_value, target, + stream); + } + // End group. + target = stream->EnsureSpace(target); + target = io::CodedOutputStream::WriteTagToArray( + WireFormatLite::kMessageSetItemEndTag, target); + return target; +} + +size_t ExtensionSet::Extension::MessageSetItemByteSize(int number) const { + if (type != WireFormatLite::TYPE_MESSAGE || is_repeated) { + // Not a valid MessageSet extension, but compute the byte size for it the + // normal way. + return ByteSize(number); + } + + if (is_cleared) return 0; + + size_t our_size = WireFormatLite::kMessageSetItemTagsSize; + + // type_id + our_size += io::CodedOutputStream::VarintSize32(number); + + // message + size_t message_size = 0; + if (is_lazy) { + message_size = lazymessage_value->ByteSizeLong(); + } else { + message_size = message_value->ByteSizeLong(); + } + + our_size += io::CodedOutputStream::VarintSize32(message_size); + our_size += message_size; + + return our_size; +} + +size_t ExtensionSet::MessageSetByteSize() const { + size_t total_size = 0; + ForEach([&total_size](int number, const Extension& ext) { + total_size += ext.MessageSetItemByteSize(number); + }); + return total_size; +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/third_party/protobuf-lite/google/protobuf/io/coded_stream_inl.h b/third_party/protobuf-lite/generated_enum_util.cc similarity index 50% rename from third_party/protobuf-lite/google/protobuf/io/coded_stream_inl.h rename to third_party/protobuf-lite/generated_enum_util.cc index d95b06e0..d0c25a96 100644 --- a/third_party/protobuf-lite/google/protobuf/io/coded_stream_inl.h +++ b/third_party/protobuf-lite/generated_enum_util.cc @@ -28,63 +28,68 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Author: jasonh@google.com (Jason Hsueh) -// -// Implements methods of coded_stream.h that need to be inlined for performance -// reasons, but should not be defined in a public header. +#include -#ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_INL_H__ -#define GOOGLE_PROTOBUF_IO_CODED_STREAM_INL_H__ +#include -#include -#include -#include -#include -#include -#include +#include namespace google { namespace protobuf { -namespace io { +namespace internal { +namespace { -inline bool CodedInputStream::InternalReadStringInline(string* buffer, - int size) { - if (size < 0) return false; // security: size is often user-supplied +bool EnumCompareByName(const EnumEntry& a, const EnumEntry& b) { + return StringPiece(a.name) < StringPiece(b.name); +} - if (BufferSize() >= size) { - STLStringResizeUninitialized(buffer, size); - std::pair z = as_string_data(buffer); - if (z.second) { - // Oddly enough, memcpy() requires its first two args to be non-NULL even - // if we copy 0 bytes. So, we have ensured that z.first is non-NULL here. - GOOGLE_DCHECK(z.first != NULL); - memcpy(z.first, buffer_, size); - Advance(size); - } - return true; +// Gets the numeric value of the EnumEntry at the given index, but returns a +// special value for the index -1. This gives a way to use std::lower_bound on a +// sorted array of indices while searching for value that we associate with -1. +int GetValue(const EnumEntry* enums, int i, int target) { + if (i == -1) { + return target; + } else { + return enums[i].value; } - - return ReadStringFallback(buffer, size); } -inline bool CodedInputStream::InternalReadRawInline(void* buffer, int size) { - int current_buffer_size; - while ((current_buffer_size = BufferSize()) < size) { - // Reading past end of buffer. Copy what we have, then refresh. - memcpy(buffer, buffer_, current_buffer_size); - buffer = reinterpret_cast(buffer) + current_buffer_size; - size -= current_buffer_size; - Advance(current_buffer_size); - if (!Refresh()) return false; +} // namespace + +bool LookUpEnumValue(const EnumEntry* enums, size_t size, + StringPiece name, int* value) { + EnumEntry target{name, 0}; + auto it = std::lower_bound(enums, enums + size, target, EnumCompareByName); + if (it != enums + size && it->name == name) { + *value = it->value; + return true; } + return false; +} - memcpy(buffer, buffer_, size); - Advance(size); +int LookUpEnumName(const EnumEntry* enums, const int* sorted_indices, + size_t size, int value) { + auto comparator = [enums, value](int a, int b) { + return GetValue(enums, a, value) < GetValue(enums, b, value); + }; + auto it = + std::lower_bound(sorted_indices, sorted_indices + size, -1, comparator); + if (it != sorted_indices + size && enums[*it].value == value) { + return it - sorted_indices; + } + return -1; +} +bool InitializeEnumStrings( + const EnumEntry* enums, const int* sorted_indices, size_t size, + internal::ExplicitlyConstructed* enum_strings) { + for (int i = 0; i < size; ++i) { + enum_strings[i].Construct(enums[sorted_indices[i]].name); + internal::OnShutdownDestroyString(enum_strings[i].get_mutable()); + } return true; } -} // namespace io +} // namespace internal } // namespace protobuf } // namespace google -#endif // GOOGLE_PROTOBUF_IO_CODED_STREAM_INL_H__ diff --git a/third_party/protobuf-lite/generated_message_table_driven_lite.cc b/third_party/protobuf-lite/generated_message_table_driven_lite.cc index 961329f3..02e6dace 100644 --- a/third_party/protobuf-lite/generated_message_table_driven_lite.cc +++ b/third_party/protobuf-lite/generated_message_table_driven_lite.cc @@ -36,7 +36,6 @@ #include #include #include -#include namespace google { namespace protobuf { @@ -44,40 +43,40 @@ namespace internal { namespace { -string* MutableUnknownFields(MessageLite* msg, int64 arena_offset) { - return Raw(msg, arena_offset) - ->mutable_unknown_fields(); +std::string* MutableUnknownFields(MessageLite* msg, int64 arena_offset) { + return Raw(msg, arena_offset) + ->mutable_unknown_fields(); } struct UnknownFieldHandlerLite { + // TODO(mvels): consider renaming UnknownFieldHandler to (TableDrivenTraits?), + // and conflating InternalMetaData into it, simplifying the template. + static constexpr bool IsLite() { return true; } + static bool Skip(MessageLite* msg, const ParseTable& table, - io::CodedInputStream* input, - int tag) { + io::CodedInputStream* input, int tag) { GOOGLE_DCHECK(!table.unknown_field_set); - ::google::protobuf::io::StringOutputStream unknown_fields_string( + io::StringOutputStream unknown_fields_string( MutableUnknownFields(msg, table.arena_offset)); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_string, false); + io::CodedOutputStream unknown_fields_stream(&unknown_fields_string, false); - return ::google::protobuf::internal::WireFormatLite::SkipField( - input, tag, &unknown_fields_stream); + return internal::WireFormatLite::SkipField(input, tag, + &unknown_fields_stream); } - static void Varint(MessageLite* msg, const ParseTable& table, - int tag, int value) { + static void Varint(MessageLite* msg, const ParseTable& table, int tag, + int value) { GOOGLE_DCHECK(!table.unknown_field_set); - ::google::protobuf::io::StringOutputStream unknown_fields_string( + io::StringOutputStream unknown_fields_string( MutableUnknownFields(msg, table.arena_offset)); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_string, false); + io::CodedOutputStream unknown_fields_stream(&unknown_fields_string, false); unknown_fields_stream.WriteVarint32(tag); unknown_fields_stream.WriteVarint32(value); } - static bool ParseExtension( - MessageLite* msg, const ParseTable& table, - io::CodedInputStream* input, int tag) { + static bool ParseExtension(MessageLite* msg, const ParseTable& table, + io::CodedInputStream* input, int tag) { ExtensionSet* extensions = GetExtensionSet(msg, table.extension_offset); if (extensions == NULL) { return false; @@ -86,22 +85,20 @@ struct UnknownFieldHandlerLite { const MessageLite* prototype = table.default_instance(); GOOGLE_DCHECK(!table.unknown_field_set); - ::google::protobuf::io::StringOutputStream unknown_fields_string( + io::StringOutputStream unknown_fields_string( MutableUnknownFields(msg, table.arena_offset)); - ::google::protobuf::io::CodedOutputStream unknown_fields_stream( - &unknown_fields_string, false); - return extensions->ParseField( - tag, input, prototype, &unknown_fields_stream); + io::CodedOutputStream unknown_fields_stream(&unknown_fields_string, false); + return extensions->ParseField(tag, input, prototype, + &unknown_fields_stream); } }; } // namespace -bool MergePartialFromCodedStreamLite( - MessageLite* msg, const ParseTable& table, io::CodedInputStream* input) { - return MergePartialFromCodedStreamImpl( - msg, table, input); +bool MergePartialFromCodedStreamLite(MessageLite* msg, const ParseTable& table, + io::CodedInputStream* input) { + return MergePartialFromCodedStreamImpl(msg, table, + input); } } // namespace internal diff --git a/third_party/protobuf-lite/generated_message_util.cc b/third_party/protobuf-lite/generated_message_util.cc index e0241361..15eb9d65 100644 --- a/third_party/protobuf-lite/generated_message_util.cc +++ b/third_party/protobuf-lite/generated_message_util.cc @@ -35,41 +35,41 @@ #include #include + +#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP // We're only using this as a standard way for getting the thread id. // We're not using any thread functionality. #include // NOLINT +#endif // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + #include -#include #include +#include #include #include +#include #include #include #include -#include +#include #include #include -#include -namespace google { +namespace google { namespace protobuf { namespace internal { void DestroyMessage(const void* message) { static_cast(message)->~MessageLite(); } -void DestroyString(const void* s) { static_cast(s)->~string(); } +void DestroyString(const void* s) { + static_cast(s)->~string(); +} ExplicitlyConstructed fixed_address_empty_string; -double Infinity() { - return std::numeric_limits::infinity(); -} -double NaN() { - return std::numeric_limits::quiet_NaN(); -} static bool InitProtobufDefaultsImpl() { fixed_address_empty_string.DefaultConstruct(); @@ -82,7 +82,7 @@ void InitProtobufDefaults() { (void)is_inited; } -size_t StringSpaceUsedExcludingSelfLong(const string& str) { +size_t StringSpaceUsedExcludingSelfLong(const std::string& str) { const void* start = &str; const void* end = &str + 1; if (start <= str.data() && str.data() < end) { @@ -108,8 +108,7 @@ struct PrimitiveTypeHelper; template <> struct PrimitiveTypeHelper { typedef bool Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteBoolNoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -120,8 +119,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef int32 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteInt32NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -132,8 +130,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef int32 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteSInt32NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -144,8 +141,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef uint32 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteUInt32NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -155,8 +151,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef int64 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteInt64NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -167,8 +162,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef int64 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteSInt64NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -178,8 +172,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef uint64 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteUInt64NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -190,8 +183,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef uint32 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteFixed32NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -202,8 +194,7 @@ struct PrimitiveTypeHelper { template <> struct PrimitiveTypeHelper { typedef uint64 Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { WireFormatLite::WriteFixed64NoTag(Get(ptr), output); } static uint8* SerializeToArray(const void* ptr, uint8* buffer) { @@ -238,9 +229,8 @@ struct PrimitiveTypeHelper template <> struct PrimitiveTypeHelper { - typedef string Type; - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { + typedef std::string Type; + static void Serialize(const void* ptr, io::CodedOutputStream* output) { const Type& value = *static_cast(ptr); output->WriteVarint32(value.size()); output->WriteRawMaybeAliased(value.data(), value.size()); @@ -283,9 +273,8 @@ void WriteLengthTo(uint32 length, O* output) { // Specialization for coded output stream template -struct OutputHelper<::google::protobuf::io::CodedOutputStream, type> { - static void Serialize(const void* ptr, - ::google::protobuf::io::CodedOutputStream* output) { +struct OutputHelper { + static void Serialize(const void* ptr, io::CodedOutputStream* output) { PrimitiveTypeHelper::Serialize(ptr, output); } }; @@ -304,35 +293,29 @@ struct OutputHelper { }; void SerializeMessageNoTable(const MessageLite* msg, - ::google::protobuf::io::CodedOutputStream* output) { + io::CodedOutputStream* output) { msg->SerializeWithCachedSizes(output); } void SerializeMessageNoTable(const MessageLite* msg, ArrayOutput* output) { - output->ptr = msg->InternalSerializeWithCachedSizesToArray( - output->is_deterministic, output->ptr); + io::ArrayOutputStream array_stream(output->ptr, INT_MAX); + io::CodedOutputStream o(&array_stream); + o.SetSerializationDeterministic(output->is_deterministic); + msg->SerializeWithCachedSizes(&o); + output->ptr += o.ByteCount(); } // Helper to branch to fast path if possible -void SerializeMessageDispatch(const ::google::protobuf::MessageLite& msg, +void SerializeMessageDispatch(const MessageLite& msg, const FieldMetadata* field_table, int num_fields, int32 cached_size, - ::google::protobuf::io::CodedOutputStream* output) { + io::CodedOutputStream* output) { const uint8* base = reinterpret_cast(&msg); - // Try the fast path - uint8* ptr = output->GetDirectBufferForNBytesAndAdvance(cached_size); - if (ptr) { - // We use virtual dispatch to enable dedicated generated code for the - // fast path. - msg.InternalSerializeWithCachedSizesToArray( - output->IsSerializationDeterministic(), ptr); - return; - } SerializeInternal(base, field_table, num_fields, output); } // Helper to branch to fast path if possible -void SerializeMessageDispatch(const ::google::protobuf::MessageLite& msg, +void SerializeMessageDispatch(const MessageLite& msg, const FieldMetadata* field_table, int num_fields, int32 cached_size, ArrayOutput* output) { const uint8* base = reinterpret_cast(&msg); @@ -430,7 +413,7 @@ struct SingularFieldHelper { template static void Serialize(const void* field, const FieldMetadata& md, O* output) { WriteTagTo(md.tag, output); - SerializeTo(&Get<::std::string>(field), output); + SerializeTo(&Get(field), output); } }; @@ -499,8 +482,8 @@ struct RepeatedFieldHelper { for (int i = 0; i < AccessorHelper::Size(array); i++) { WriteTagTo(md.tag, output); SerializeMessageTo( - static_cast(AccessorHelper::Get(array, i)), md.ptr, - output); + static_cast(AccessorHelper::Get(array, i)), + md.ptr, output); } } }; @@ -563,7 +546,7 @@ struct OneOfFieldHelper { template static void Serialize(const void* field, const FieldMetadata& md, O* output) { SingularFieldHelper::Serialize( - Get(field), md, output); + Get(field), md, output); } }; @@ -609,7 +592,7 @@ bool IsNull(const void* ptr) { template <> bool IsNull(const void* ptr) { - return static_cast(ptr)->empty(); + return static_cast(ptr)->empty(); } #define SERIALIZERS_FOR_TYPE(type) \ @@ -635,8 +618,8 @@ bool IsNull(const void* ptr) { void SerializeInternal(const uint8* base, const FieldMetadata* field_metadata_table, - int32 num_fields, - ::google::protobuf::io::CodedOutputStream* output) { + int32 num_fields, io::CodedOutputStream* output) { + SpecialSerializer func = nullptr; for (int i = 0; i < num_fields; i++) { const FieldMetadata& field_metadata = field_metadata_table[i]; const uint8* ptr = base + field_metadata.offset; @@ -663,10 +646,10 @@ void SerializeInternal(const uint8* base, // Special cases case FieldMetadata::kSpecial: - reinterpret_cast( - const_cast(field_metadata.ptr))( - base, field_metadata.offset, field_metadata.tag, - field_metadata.has_offset, output); + func = reinterpret_cast( + const_cast(field_metadata.ptr)); + func(base, field_metadata.offset, field_metadata.tag, + field_metadata.has_offset, output); break; default: // __builtin_unreachable() @@ -681,6 +664,7 @@ uint8* SerializeInternalToArray(const uint8* base, uint8* buffer) { ArrayOutput array_output = {buffer, is_deterministic}; ArrayOutput* output = &array_output; + SpecialSerializer func = nullptr; for (int i = 0; i < num_fields; i++) { const FieldMetadata& field_metadata = field_metadata_table[i]; const uint8* ptr = base + field_metadata.offset; @@ -709,10 +693,10 @@ uint8* SerializeInternalToArray(const uint8* base, io::ArrayOutputStream array_stream(array_output.ptr, INT_MAX); io::CodedOutputStream output(&array_stream); output.SetSerializationDeterministic(is_deterministic); - reinterpret_cast( - const_cast(field_metadata.ptr))( - base, field_metadata.offset, field_metadata.tag, - field_metadata.has_offset, &output); + func = reinterpret_cast( + const_cast(field_metadata.ptr)); + func(base, field_metadata.offset, field_metadata.tag, + field_metadata.has_offset, &output); array_output.ptr += output.ByteCount(); } break; default: @@ -725,18 +709,17 @@ uint8* SerializeInternalToArray(const uint8* base, #undef SERIALIZERS_FOR_TYPE void ExtensionSerializer(const uint8* ptr, uint32 offset, uint32 tag, - uint32 has_offset, - ::google::protobuf::io::CodedOutputStream* output) { + uint32 has_offset, io::CodedOutputStream* output) { reinterpret_cast(ptr + offset) ->SerializeWithCachedSizes(tag, has_offset, output); } void UnknownFieldSerializerLite(const uint8* ptr, uint32 offset, uint32 tag, uint32 has_offset, - ::google::protobuf::io::CodedOutputStream* output) { + io::CodedOutputStream* output) { output->WriteString( - reinterpret_cast(ptr + offset) - ->unknown_fields()); + reinterpret_cast(ptr + offset) + ->unknown_fields(&internal::GetEmptyString)); } MessageLite* DuplicateIfNonNullInternal(MessageLite* message) { @@ -749,6 +732,15 @@ MessageLite* DuplicateIfNonNullInternal(MessageLite* message) { } } +void GenericSwap(MessageLite* m1, MessageLite* m2) { + std::unique_ptr tmp(m1->New()); + tmp->CheckTypeAndMergeFrom(*m1); + m1->Clear(); + m1->CheckTypeAndMergeFrom(*m2); + m2->Clear(); + m2->CheckTypeAndMergeFrom(*tmp); +} + // Returns a message owned by this Arena. This may require Own()ing or // duplicating the message. MessageLite* GetOwnedMessageInternal(Arena* message_arena, @@ -770,12 +762,22 @@ namespace { void InitSCC_DFS(SCCInfoBase* scc) { if (scc->visit_status.load(std::memory_order_relaxed) != - SCCInfoBase::kUninitialized) return; + SCCInfoBase::kUninitialized) + return; scc->visit_status.store(SCCInfoBase::kRunning, std::memory_order_relaxed); - // Each base is followed by an array of pointers to deps - auto deps = reinterpret_cast(scc + 1); - for (int i = 0; i < scc->num_deps; i++) { - if (deps[i]) InitSCC_DFS(deps[i]); + // Each base is followed by an array of void*, containing first pointers to + // SCCInfoBase and then pointers-to-pointers to SCCInfoBase. + auto deps = reinterpret_cast(scc + 1); + auto strong_deps = reinterpret_cast(deps); + for (int i = 0; i < scc->num_deps; ++i) { + if (strong_deps[i]) InitSCC_DFS(strong_deps[i]); + } + auto implicit_weak_deps = + reinterpret_cast(deps + scc->num_deps); + for (int i = 0; i < scc->num_implicit_weak_deps; ++i) { + if (*implicit_weak_deps[i]) { + InitSCC_DFS(*implicit_weak_deps[i]); + } } scc->init_func(); // Mark done (note we use memory order release here), other threads could @@ -790,8 +792,17 @@ void InitSCCImpl(SCCInfoBase* scc) { static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED}; // Either the default in case no initialization is running or the id of the // thread that is currently initializing. +#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP static std::atomic runner; auto me = std::this_thread::get_id(); +#else + // This is a lightweight replacement for std::thread::id. std::thread does not + // work on Windows XP SP2 with the latest VC++ libraries, because it utilizes + // the Concurrency Runtime that is only supported on Windows XP SP3 and above. + static std::atomic_llong runner(-1); + auto me = ::GetCurrentThreadId(); +#endif // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + // This will only happen because the constructor will call InitSCC while // constructing the default instance. if (runner.load(std::memory_order_relaxed) == me) { @@ -805,7 +816,13 @@ void InitSCCImpl(SCCInfoBase* scc) { mu.Lock(); runner.store(me, std::memory_order_relaxed); InitSCC_DFS(scc); + +#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP runner.store(std::thread::id{}, std::memory_order_relaxed); +#else + runner.store(-1, std::memory_order_relaxed); +#endif // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + mu.Unlock(); } diff --git a/third_party/protobuf-lite/google/protobuf/arena.h b/third_party/protobuf-lite/google/protobuf/arena.h index 9928c8e6..038553c8 100644 --- a/third_party/protobuf-lite/google/protobuf/arena.h +++ b/third_party/protobuf-lite/google/protobuf/arena.h @@ -33,7 +33,10 @@ #ifndef GOOGLE_PROTOBUF_ARENA_H__ #define GOOGLE_PROTOBUF_ARENA_H__ + #include +#include +#include #ifdef max #undef max // Visual Studio defines this macro #endif @@ -48,9 +51,15 @@ using type_info = ::type_info; #include #endif -#include -#include #include +#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { @@ -58,29 +67,28 @@ namespace protobuf { struct ArenaOptions; // defined below } // namespace protobuf +} // namespace google -namespace quality_webanswers { - -void TempPrivateWorkAround(::google::protobuf::ArenaOptions* arena_options); - -} // namespace quality_webanswers - +namespace google { namespace protobuf { -class Arena; // defined below -class Message; // defined in message.h +class Arena; // defined below +class Message; // defined in message.h class MessageLite; +template +class Map; namespace arena_metrics { -void EnableArenaMetrics(::google::protobuf::ArenaOptions* options); +void EnableArenaMetrics(ArenaOptions* options); } // namespace arena_metrics namespace internal { -struct ArenaStringPtr; // defined in arenastring.h -class LazyField; // defined in lazy_field.h +struct ArenaStringPtr; // defined in arenastring.h +class LazyField; // defined in lazy_field.h +class EpsCopyInputStream; // defined in parse_context.h template class GenericTypeHandler; // defined in repeated_field.h @@ -153,13 +161,14 @@ struct ArenaOptions { private: // Hooks for adding external functionality such as user-specific metrics // collection, specific debugging abilities, etc. - // Init hook may return a pointer to a cookie to be stored in the arena. - // reset and destruction hooks will then be called with the same cookie - // pointer. This allows us to save an external object per arena instance and - // use it on the other hooks (Note: It is just as legal for init to return - // NULL and not use the cookie feature). - // on_arena_reset and on_arena_destruction also receive the space used in - // the arena just before the reset. + // Init hook (if set) will always be called at Arena init time. Init hook may + // return a pointer to a cookie to be stored in the arena. Reset and + // destruction hooks will then be called with the same cookie pointer. This + // allows us to save an external object per arena instance and use it on the + // other hooks (Note: If init hook returns NULL, the other hooks will NOT be + // called on this arena instance). + // on_arena_reset and on_arena_destruction also receive the space used in the + // arena just before the reset. void* (*on_arena_init)(Arena* arena); void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used); void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used); @@ -177,15 +186,14 @@ struct ArenaOptions { static const size_t kDefaultStartBlockSize = 256; static const size_t kDefaultMaxBlockSize = 8192; - friend void ::google::protobuf::arena_metrics::EnableArenaMetrics(ArenaOptions*); - friend void quality_webanswers::TempPrivateWorkAround(ArenaOptions*); + friend void arena_metrics::EnableArenaMetrics(ArenaOptions*); friend class Arena; friend class ArenaOptionsTestFriend; }; // Support for non-RTTI environments. (The metrics hooks API uses type // information.) -#ifndef GOOGLE_PROTOBUF_NO_RTTI +#if PROTOBUF_RTTI #define RTTI_TYPE_ID(type) (&typeid(type)) #else #define RTTI_TYPE_ID(type) (NULL) @@ -211,14 +219,15 @@ struct ArenaOptions { // any special requirements on the type T, and will invoke the object's // destructor when the arena is destroyed. // -// The arena message allocation protocol, required by CreateMessage, is as -// follows: +// The arena message allocation protocol, required by +// CreateMessage(Arena* arena, Args&&... args), is as follows: // -// - The type T must have (at least) two constructors: a constructor with no -// arguments, called when a T is allocated on the heap; and a constructor with -// a google::protobuf::Arena* argument, called when a T is allocated on an arena. If the -// second constructor is called with a NULL arena pointer, it must be -// equivalent to invoking the first (no-argument) constructor. +// - The type T must have (at least) two constructors: a constructor callable +// with `args` (without `arena`), called when a T is allocated on the heap; +// and a constructor callable with `Arena* arena, Args&&... args`, called when +// a T is allocated on an arena. If the second constructor is called with a +// NULL arena pointer, it must be equivalent to invoking the first +// (`args`-only) constructor. // // - The type T must have a particular type trait: a nested type // |InternalArenaConstructable_|. This is usually a typedef to |void|. If no @@ -231,17 +240,11 @@ struct ArenaOptions { // present on the type, then its destructor is always called when the // containing arena is destroyed. // -// - One- and two-user-argument forms of CreateMessage() also exist that -// forward these constructor arguments to T's constructor: for example, -// CreateMessage(Arena*, arg1, arg2) forwards to a constructor T(Arena*, -// arg1, arg2). -// // This protocol is implemented by all arena-enabled proto2 message classes as -// well as RepeatedPtrField. -// -// Do NOT subclass Arena. This class will be marked as final when C++11 is -// enabled. -class LIBPROTOBUF_EXPORT Arena { +// well as protobuf container types like RepeatedPtrField and Map. The protocol +// is internal to protobuf and is not guaranteed to be stable. Non-proto types +// should not rely on this protocol. +class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { public: // Arena constructor taking custom options. See ArenaOptions below for // descriptions of the options available. @@ -291,8 +294,7 @@ class LIBPROTOBUF_EXPORT Arena { // This function also accepts any type T that satisfies the arena message // allocation protocol, documented above. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateMessage( - Arena* arena, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* CreateMessage(Arena* arena, Args&&... args) { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); @@ -318,8 +320,7 @@ class LIBPROTOBUF_EXPORT Arena { // if the object were allocated on the heap (except that the underlying memory // is obtained from the arena). template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* Create(Arena* arena, - Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* Create(Arena* arena, Args&&... args) { return CreateNoMessage(arena, is_arena_constructable(), std::forward(args)...); } @@ -331,8 +332,8 @@ class LIBPROTOBUF_EXPORT Arena { // (when compiled as C++11) that T is trivially default-constructible and // trivially destructible. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateArray( - Arena* arena, size_t num_elements) { + PROTOBUF_ALWAYS_INLINE static T* CreateArray(Arena* arena, + size_t num_elements) { static_assert(std::is_pod::value, "CreateArray requires a trivially constructible type"); static_assert(std::is_trivially_destructible::value, @@ -355,21 +356,13 @@ class LIBPROTOBUF_EXPORT Arena { // may not include space used by other threads executing concurrently with // the call to this method. uint64 SpaceUsed() const { return impl_.SpaceUsed(); } - // DEPRECATED. Please use SpaceAllocated() and SpaceUsed(). - // - // Combines SpaceAllocated and SpaceUsed. Returns a pair of - // . - PROTOBUF_RUNTIME_DEPRECATED("Please use SpaceAllocated() and SpaceUsed()") - std::pair SpaceAllocatedAndUsed() const { - return std::make_pair(SpaceAllocated(), SpaceUsed()); - } // Frees all storage allocated by this arena after calling destructors // registered with OwnDestructor() and freeing objects registered with Own(). // Any objects allocated on this arena are unusable after this call. It also // returns the total space used by the arena which is the sums of the sizes // of the allocated blocks. This method is not thread-safe. - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE uint64 Reset() { + PROTOBUF_NOINLINE uint64 Reset() { // Call the reset hook if (on_arena_reset_ != NULL) { on_arena_reset_(this, hooks_cookie_, impl_.SpaceAllocated()); @@ -380,7 +373,7 @@ class LIBPROTOBUF_EXPORT Arena { // Adds |object| to a list of heap-allocated objects to be freed with |delete| // when the arena is destroyed or reset. template - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE void Own(T* object) { + PROTOBUF_NOINLINE void Own(T* object) { OwnInternal(object, std::is_convertible()); } @@ -390,7 +383,7 @@ class LIBPROTOBUF_EXPORT Arena { // normally only used for objects that are placement-newed into // arena-allocated memory. template - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE void OwnDestructor(T* object) { + PROTOBUF_NOINLINE void OwnDestructor(T* object) { if (object != NULL) { impl_.AddCleanup(object, &internal::arena_destruct_object); } @@ -400,19 +393,18 @@ class LIBPROTOBUF_EXPORT Arena { // will be manually called when the arena is destroyed or reset. This differs // from OwnDestructor() in that any member function may be specified, not only // the class destructor. - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE void OwnCustomDestructor( - void* object, void (*destruct)(void*)) { + PROTOBUF_NOINLINE void OwnCustomDestructor(void* object, + void (*destruct)(void*)) { impl_.AddCleanup(object, destruct); } // Retrieves the arena associated with |value| if |value| is an arena-capable - // message, or NULL otherwise. This differs from value->GetArena() in that the - // latter is a virtual call, while this method is a templated call that - // resolves at compile-time. + // message, or NULL otherwise. If possible, the call resolves at compile time. + // Note that we can often devirtualize calls to `value->GetArena()` so usually + // calling this method is unnecessary. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static Arena* GetArena( - const T* value) { - return GetArenaInternal(value, is_arena_constructable()); + PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) { + return GetArenaInternal(value); } template @@ -439,12 +431,25 @@ class LIBPROTOBUF_EXPORT Arena { sizeof(char)> is_arena_constructable; + template () + .GetArena())>::value, + int>::type = 0> + static char HasGetArena(decltype(&U::GetArena)); + template + static double HasGetArena(...); + + typedef std::integral_constant(nullptr)) == + sizeof(char)> + has_get_arena; + template static T* Construct(void* ptr, Args&&... args) { return new (ptr) T(std::forward(args)...); } - static Arena* GetArena(const T* p) { return p->GetArenaNoVirtual(); } + static Arena* GetArena(const T* p) { return p->GetArena(); } friend class Arena; }; @@ -468,9 +473,12 @@ class LIBPROTOBUF_EXPORT Arena { }; private: + template + struct has_get_arena : InternalHelper::has_get_arena {}; + template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateMessageInternal( - Arena* arena, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* CreateMessageInternal(Arena* arena, + Args&&... args) { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); @@ -485,8 +493,7 @@ class LIBPROTOBUF_EXPORT Arena { // slightly different. When the arena pointer is nullptr, it calls T() // instead of T(nullptr). template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateMessageInternal( - Arena* arena) { + PROTOBUF_ALWAYS_INLINE static T* CreateMessageInternal(Arena* arena) { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); @@ -498,8 +505,8 @@ class LIBPROTOBUF_EXPORT Arena { } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateInternal( - Arena* arena, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, + Args&&... args) { if (arena == NULL) { return new T(std::forward(args)...); } else { @@ -511,7 +518,7 @@ class LIBPROTOBUF_EXPORT Arena { void CallDestructorHooks(); void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const; inline void AllocHook(const std::type_info* allocated_type, size_t n) const { - if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL)) { + if (PROTOBUF_PREDICT_FALSE(hooks_cookie_ != NULL)) { OnArenaAllocation(allocated_type, n); } } @@ -520,13 +527,13 @@ class LIBPROTOBUF_EXPORT Arena { // allocated type info when the hooks are in place in ArenaOptions and // the cookie is not null. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void* AllocateInternal( - bool skip_explicit_ownership) { + PROTOBUF_ALWAYS_INLINE void* AllocateInternal(bool skip_explicit_ownership) { + static_assert(alignof(T) <= 8, "T is overaligned, see b/151247138"); const size_t n = internal::AlignUpTo8(sizeof(T)); AllocHook(RTTI_TYPE_ID(T), n); // Monitor allocation if needed. if (skip_explicit_ownership) { - return impl_.AllocateAligned(n); + return AllocateAlignedNoHook(n); } else { return impl_.AllocateAlignedAndAddCleanup( n, &internal::arena_destruct_object); @@ -539,27 +546,29 @@ class LIBPROTOBUF_EXPORT Arena { // user code. These are used only internally from LazyField and Repeated // fields, since they are designed to work in all mode combinations. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static Msg* DoCreateMaybeMessage( - Arena* arena, std::true_type, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static Msg* DoCreateMaybeMessage(Arena* arena, + std::true_type, + Args&&... args) { return CreateMessageInternal(arena, std::forward(args)...); } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* DoCreateMaybeMessage( - Arena* arena, std::false_type, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* DoCreateMaybeMessage(Arena* arena, + std::false_type, + Args&&... args) { return CreateInternal(arena, std::forward(args)...); } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateMaybeMessage( - Arena* arena, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena, + Args&&... args) { return DoCreateMaybeMessage(arena, is_arena_constructable(), std::forward(args)...); } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateNoMessage( - Arena* arena, std::true_type, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* CreateNoMessage(Arena* arena, std::true_type, + Args&&... args) { // User is constructing with Create() despite the fact that T supports arena // construction. In this case we have to delegate to CreateInternal(), and // we can't use any CreateMaybeMessage() specialization that may be defined. @@ -567,8 +576,9 @@ class LIBPROTOBUF_EXPORT Arena { } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateNoMessage( - Arena* arena, std::false_type, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* CreateNoMessage(Arena* arena, + std::false_type, + Args&&... args) { // User is constructing with Create() and the type does not support arena // construction. In this case we can delegate to CreateMaybeMessage() and // use any specialization that may be available for that. @@ -578,50 +588,50 @@ class LIBPROTOBUF_EXPORT Arena { // Just allocate the required size for the given type assuming the // type has a trivial constructor. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* CreateInternalRawArray( - size_t num_elements) { + PROTOBUF_ALWAYS_INLINE T* CreateInternalRawArray(size_t num_elements) { GOOGLE_CHECK_LE(num_elements, std::numeric_limits::max() / sizeof(T)) << "Requested size is too large to fit into size_t."; const size_t n = internal::AlignUpTo8(sizeof(T) * num_elements); // Monitor allocation if needed. AllocHook(RTTI_TYPE_ID(T), n); - return static_cast(impl_.AllocateAligned(n)); + return static_cast(AllocateAlignedNoHook(n)); } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* DoCreate( - bool skip_explicit_ownership, Args&&... args) { + PROTOBUF_ALWAYS_INLINE T* DoCreate(bool skip_explicit_ownership, + Args&&... args) { return new (AllocateInternal(skip_explicit_ownership)) T(std::forward(args)...); } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* DoCreateMessage(Args&&... args) { + PROTOBUF_ALWAYS_INLINE T* DoCreateMessage(Args&&... args) { return InternalHelper::Construct( AllocateInternal(InternalHelper::is_destructor_skippable::value), this, std::forward(args)...); } // CreateInArenaStorage is used to implement map field. Without it, - // google::protobuf::Map need to call generated message's protected arena constructor, - // which needs to declare google::protobuf::Map as friend of generated message. - template - static void CreateInArenaStorage(T* ptr, Arena* arena) { + // Map need to call generated message's protected arena constructor, + // which needs to declare Map as friend of generated message. + template + static void CreateInArenaStorage(T* ptr, Arena* arena, Args&&... args) { CreateInArenaStorageInternal(ptr, arena, - typename is_arena_constructable::type()); + typename is_arena_constructable::type(), + std::forward(args)...); RegisterDestructorInternal( ptr, arena, typename InternalHelper::is_destructor_skippable::type()); } - template + template static void CreateInArenaStorageInternal(T* ptr, Arena* arena, - std::true_type) { - InternalHelper::Construct(ptr, arena); + std::true_type, Args&&... args) { + InternalHelper::Construct(ptr, arena, std::forward(args)...); } - template + template static void CreateInArenaStorageInternal(T* ptr, Arena* /* arena */, - std::false_type) { - new (ptr) T(); + std::false_type, Args&&... args) { + new (ptr) T(std::forward(args)...); } template @@ -635,19 +645,17 @@ class LIBPROTOBUF_EXPORT Arena { // These implement Own(), which registers an object for deletion (destructor // call and operator delete()). The second parameter has type 'true_type' if T - // is a subtype of ::google::protobuf::Message and 'false_type' otherwise. Collapsing + // is a subtype of Message and 'false_type' otherwise. Collapsing // all template instantiations to one for generic Message reduces code size, // using the virtual destructor instead. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void OwnInternal(T* object, - std::true_type) { + PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) { if (object != NULL) { impl_.AddCleanup(object, &internal::arena_delete_object); } } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void OwnInternal(T* object, - std::false_type) { + PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) { if (object != NULL) { impl_.AddCleanup(object, &internal::arena_delete_object); } @@ -655,25 +663,47 @@ class LIBPROTOBUF_EXPORT Arena { // Implementation for GetArena(). Only message objects with // InternalArenaConstructable_ tags can be associated with an arena, and such - // objects must implement a GetArenaNoVirtual() method. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static Arena* GetArenaInternal( - const T* value, std::true_type) { + // objects must implement a GetArena() method. + template ::value, int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { return InternalHelper::GetArena(value); } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static Arena* GetArenaInternal( - const T* /* value */, std::false_type) { - return NULL; + template ::value && + has_get_arena::value, + int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { + return value->GetArena(); + } + template ::value && + !has_get_arena::value, + int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { + (void)value; + return nullptr; } // For friends of arena. void* AllocateAligned(size_t n) { AllocHook(NULL, n); - return impl_.AllocateAligned(internal::AlignUpTo8(n)); + return AllocateAlignedNoHook(internal::AlignUpTo8(n)); + } + template + void* AllocateAlignedTo(size_t n) { + static_assert(Align > 0, "Alignment must be greater than 0"); + static_assert((Align & (Align - 1)) == 0, "Alignment must be power of two"); + if (Align <= 8) return AllocateAligned(n); + // TODO(b/151247138): if the pointer would have been aligned already, + // this is wasting space. We should pass the alignment down. + uintptr_t ptr = reinterpret_cast(AllocateAligned(n + Align - 8)); + ptr = (ptr + Align - 1) & -Align; + return reinterpret_cast(ptr); } + void* AllocateAlignedNoHook(size_t n); + internal::ArenaImpl impl_; void (*on_arena_allocation_)(const std::type_info* allocated_type, @@ -689,6 +719,7 @@ class LIBPROTOBUF_EXPORT Arena { friend class internal::GenericTypeHandler; friend struct internal::ArenaStringPtr; // For AllocateAligned. friend class internal::LazyField; // For CreateMaybeMessage. + friend class internal::EpsCopyInputStream; // For parser performance friend class MessageLite; template friend class Map; @@ -698,6 +729,8 @@ class LIBPROTOBUF_EXPORT Arena { #undef RTTI_TYPE_ID } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENA_H__ diff --git a/third_party/protobuf-lite/google/protobuf/arena_impl.h b/third_party/protobuf-lite/google/protobuf/arena_impl.h index f648f166..5a95124c 100644 --- a/third_party/protobuf-lite/google/protobuf/arena_impl.h +++ b/third_party/protobuf-lite/google/protobuf/arena_impl.h @@ -39,29 +39,31 @@ #include #include -#include - #ifdef ADDRESS_SANITIZER #include #endif // ADDRESS_SANITIZER -namespace google { +#include + +namespace google { namespace protobuf { namespace internal { inline size_t AlignUpTo8(size_t n) { // Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.) - return (n + 7) & -8; + return (n + 7) & static_cast(-8); } +using LifecycleId = int64_t; + // This class provides the core Arena memory allocation library. Different // implementations only need to implement the public interface below. // Arena is not a template type as that would only be useful if all protos // in turn would be templates, which will/cannot happen. However separating // the memory allocation part from the cruft of the API users expect we can // use #ifdef the select the best implementation based on hardware / OS. -class LIBPROTOBUF_EXPORT ArenaImpl { +class PROTOBUF_EXPORT ArenaImpl { public: struct Options { size_t start_block_size; @@ -73,12 +75,12 @@ class LIBPROTOBUF_EXPORT ArenaImpl { template explicit Options(const O& options) - : start_block_size(options.start_block_size), - max_block_size(options.max_block_size), - initial_block(options.initial_block), - initial_block_size(options.initial_block_size), - block_alloc(options.block_alloc), - block_dealloc(options.block_dealloc) {} + : start_block_size(options.start_block_size), + max_block_size(options.max_block_size), + initial_block(options.initial_block), + initial_block_size(options.initial_block_size), + block_alloc(options.block_alloc), + block_dealloc(options.block_dealloc) {} }; template @@ -105,7 +107,27 @@ class LIBPROTOBUF_EXPORT ArenaImpl { uint64 SpaceAllocated() const; uint64 SpaceUsed() const; - void* AllocateAligned(size_t n); + void* AllocateAligned(size_t n) { + SerialArena* arena; + if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) { + return arena->AllocateAligned(n); + } else { + return AllocateAlignedFallback(n); + } + } + + // This function allocates n bytes if the common happy case is true and + // returns true. Otherwise does nothing and returns false. This strange + // semantics is necessary to allow callers to program functions that only + // have fallback function calls in tail position. This substantially improves + // code for the happy path. + PROTOBUF_ALWAYS_INLINE bool MaybeAllocateAligned(size_t n, void** out) { + SerialArena* a; + if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFromThreadCache(&a))) { + return a->MaybeAllocateAligned(n, out); + } + return false; + } void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)); @@ -113,6 +135,8 @@ class LIBPROTOBUF_EXPORT ArenaImpl { void AddCleanup(void* elem, void (*cleanup)(void*)); private: + friend class ArenaBenchmark; + void* AllocateAlignedFallback(size_t n); void* AllocateAlignedAndAddCleanupFallback(size_t n, void (*cleanup)(void*)); void AddCleanupFallback(void* elem, void (*cleanup)(void*)); @@ -137,7 +161,7 @@ class LIBPROTOBUF_EXPORT ArenaImpl { class Block; // A thread-unsafe Arena that can only be used within its owning thread. - class LIBPROTOBUF_EXPORT SerialArena { + class PROTOBUF_EXPORT SerialArena { public: // The allocate/free methods here are a little strange, since SerialArena is // allocated inside a Block which it also manages. This is to avoid doing @@ -154,10 +178,12 @@ class LIBPROTOBUF_EXPORT ArenaImpl { void CleanupList(); uint64 SpaceUsed() const; + bool HasSpace(size_t n) { return n <= static_cast(limit_ - ptr_); } + void* AllocateAligned(size_t n) { GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. GOOGLE_DCHECK_GE(limit_, ptr_); - if (GOOGLE_PREDICT_FALSE(static_cast(limit_ - ptr_) < n)) { + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) { return AllocateAlignedFallback(n); } void* ret = ptr_; @@ -168,8 +194,22 @@ class LIBPROTOBUF_EXPORT ArenaImpl { return ret; } + // Allocate space if the current region provides enough space. + bool MaybeAllocateAligned(size_t n, void** out) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + GOOGLE_DCHECK_GE(limit_, ptr_); + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) return false; + void* ret = ptr_; + ptr_ += n; +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(ret, n); +#endif // ADDRESS_SANITIZER + *out = ret; + return true; + } + void AddCleanup(void* elem, void (*cleanup)(void*)) { - if (GOOGLE_PREDICT_FALSE(cleanup_ptr_ == cleanup_limit_)) { + if (PROTOBUF_PREDICT_FALSE(cleanup_ptr_ == cleanup_limit_)) { AddCleanupFallback(elem, cleanup); return; } @@ -193,11 +233,11 @@ class LIBPROTOBUF_EXPORT ArenaImpl { void AddCleanupFallback(void* elem, void (*cleanup)(void*)); void CleanupListFallback(); - ArenaImpl* arena_; // Containing arena. - void* owner_; // &ThreadCache of this thread; - Block* head_; // Head of linked list of blocks. - CleanupChunk* cleanup_; // Head of cleanup list. - SerialArena* next_; // Next SerialArena in this linked list. + ArenaImpl* arena_; // Containing arena. + void* owner_; // &ThreadCache of this thread; + Block* head_; // Head of linked list of blocks. + CleanupChunk* cleanup_; // Head of cleanup list. + SerialArena* next_; // Next SerialArena in this linked list. // Next pointer to allocate from. Always 8-byte aligned. Points inside // head_ (and head_->pos will always be non-canonical). We keep these @@ -212,7 +252,7 @@ class LIBPROTOBUF_EXPORT ArenaImpl { // Blocks are variable length malloc-ed objects. The following structure // describes the common header for all blocks. - class LIBPROTOBUF_EXPORT Block { + class PROTOBUF_EXPORT Block { public: Block(size_t size, Block* next); @@ -227,7 +267,7 @@ class LIBPROTOBUF_EXPORT ArenaImpl { void set_pos(size_t pos) { pos_ = pos; } private: - Block* next_; // Next block for this thread. + Block* next_; // Next block for this thread. size_t pos_; size_t size_; // data follows @@ -243,10 +283,10 @@ class LIBPROTOBUF_EXPORT ArenaImpl { // The ThreadCache is considered valid as long as this matches the // lifecycle_id of the arena being used. - int64 last_lifecycle_id_seen; + LifecycleId last_lifecycle_id_seen; SerialArena* last_serial_arena; }; - static std::atomic lifecycle_id_generator_; + static std::atomic lifecycle_id_generator_; #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) // Android ndk does not support GOOGLE_THREAD_LOCAL keyword so we use a custom thread // local storage class we implemented. @@ -279,21 +319,45 @@ class LIBPROTOBUF_EXPORT ArenaImpl { hint_.store(serial, std::memory_order_release); } - std::atomic threads_; // Pointer to a linked list of SerialArena. std::atomic hint_; // Fast thread-local block access std::atomic space_allocated_; // Total size of all allocated blocks. - Block *initial_block_; // If non-NULL, points to the block that came from - // user data. + Block* initial_block_; // If non-NULL, points to the block that came from + // user data. Block* NewBlock(Block* last_block, size_t min_bytes); SerialArena* GetSerialArena(); - bool GetSerialArenaFast(SerialArena** arena); + PROTOBUF_ALWAYS_INLINE bool GetSerialArenaFast(SerialArena** arena) { + if (GetSerialArenaFromThreadCache(arena)) return true; + + // Check whether we own the last accessed SerialArena on this arena. This + // fast path optimizes the case where a single thread uses multiple arenas. + ThreadCache* tc = &thread_cache(); + SerialArena* serial = hint_.load(std::memory_order_acquire); + if (PROTOBUF_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) { + *arena = serial; + return true; + } + return false; + } + + PROTOBUF_ALWAYS_INLINE bool GetSerialArenaFromThreadCache( + SerialArena** arena) { + // If this thread already owns a block in this arena then try to use that. + // This fast path optimizes the case where multiple threads allocate from + // the same arena. + ThreadCache* tc = &thread_cache(); + if (PROTOBUF_PREDICT_TRUE(tc->last_lifecycle_id_seen == lifecycle_id_)) { + *arena = tc->last_serial_arena; + return true; + } + return false; + } SerialArena* GetSerialArenaFallback(void* me); - int64 lifecycle_id_; // Unique for each arena. Changes on Reset(). + LifecycleId lifecycle_id_; // Unique for each arena. Changes on Reset(). Options options_; @@ -306,8 +370,10 @@ class LIBPROTOBUF_EXPORT ArenaImpl { public: // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 // to protect the invariant that pos is always at a multiple of 8. - static const size_t kBlockHeaderSize = (sizeof(Block) + 7) & -8; - static const size_t kSerialArenaSize = (sizeof(SerialArena) + 7) & -8; + static const size_t kBlockHeaderSize = + (sizeof(Block) + 7) & static_cast(-8); + static const size_t kSerialArenaSize = + (sizeof(SerialArena) + 7) & static_cast(-8); static_assert(kBlockHeaderSize % 8 == 0, "kBlockHeaderSize must be a multiple of 8."); static_assert(kSerialArenaSize % 8 == 0, @@ -316,6 +382,8 @@ class LIBPROTOBUF_EXPORT ArenaImpl { } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENA_IMPL_H__ diff --git a/third_party/protobuf-lite/google/protobuf/arenastring.h b/third_party/protobuf-lite/google/protobuf/arenastring.h index 168fc972..122f391e 100644 --- a/third_party/protobuf-lite/google/protobuf/arenastring.h +++ b/third_party/protobuf-lite/google/protobuf/arenastring.h @@ -32,20 +32,29 @@ #define GOOGLE_PROTOBUF_ARENASTRING_H__ #include +#include +#include -#include +#include #include #include -#include -#include +#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + // This is the implementation of arena string fields written for the open-source // release. The ArenaStringPtr struct below is an internal implementation class // and *should not be used* by user code. It is used to collect string // operations together into one place and abstract away the underlying // string-field pointer representation, so that (for example) an alternate -// implementation that knew more about ::std::string's internals could integrate more -// closely with the arena allocator. +// implementation that knew more about ::std::string's internals could integrate +// more closely with the arena allocator. namespace google { namespace protobuf { @@ -63,9 +72,9 @@ class TaggedPtr { uintptr_t ptr_; }; -struct LIBPROTOBUF_EXPORT ArenaStringPtr { +struct PROTOBUF_EXPORT ArenaStringPtr { inline void Set(const ::std::string* default_value, - const ::std::string& value, ::google::protobuf::Arena* arena) { + const ::std::string& value, Arena* arena) { if (ptr_ == default_value) { CreateInstance(arena, &value); } else { @@ -74,8 +83,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { } inline void SetLite(const ::std::string* default_value, - const ::std::string& value, - ::google::protobuf::Arena* arena) { + const ::std::string& value, Arena* arena) { Set(default_value, value, arena); } @@ -83,7 +91,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { inline const ::std::string& Get() const { return *ptr_; } inline ::std::string* Mutable(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { + Arena* arena) { if (ptr_ == default_value) { CreateInstance(arena, default_value); } @@ -95,7 +103,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { // retains ownership. Clears this field back to NULL state. Used to implement // release_() methods on generated classes. inline ::std::string* Release(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { + Arena* arena) { if (ptr_ == default_value) { return NULL; } @@ -103,8 +111,8 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { } // Similar to Release, but ptr_ cannot be the default_value. - inline ::std::string* ReleaseNonDefault( - const ::std::string* default_value, ::google::protobuf::Arena* arena) { + inline ::std::string* ReleaseNonDefault(const ::std::string* default_value, + Arena* arena) { GOOGLE_DCHECK(!IsDefault(default_value)); ::std::string* released = NULL; if (arena != NULL) { @@ -114,22 +122,22 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { } else { released = ptr_; } - ptr_ = const_cast< ::std::string* >(default_value); + ptr_ = const_cast< ::std::string*>(default_value); return released; } - // UnsafeArenaRelease returns a ::std::string*, but it may be arena-owned (i.e. - // have its destructor already registered) if arena != NULL. If the field was - // not set, this returns NULL. This method clears this field back to NULL - // state. Used to implement unsafe_arena_release_() methods on + // UnsafeArenaRelease returns a ::std::string*, but it may be arena-owned + // (i.e. have its destructor already registered) if arena != NULL. If the + // field was not set, this returns NULL. This method clears this field back to + // NULL state. Used to implement unsafe_arena_release_() methods on // generated classes. inline ::std::string* UnsafeArenaRelease(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { + Arena* /* arena */) { if (ptr_ == default_value) { return NULL; } ::std::string* released = ptr_; - ptr_ = const_cast< ::std::string* >(default_value); + ptr_ = const_cast< ::std::string*>(default_value); return released; } @@ -137,7 +145,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { // destructor is registered with the arena. Used to implement // set_allocated_ in generated classes. inline void SetAllocated(const ::std::string* default_value, - ::std::string* value, ::google::protobuf::Arena* arena) { + ::std::string* value, Arena* arena) { if (arena == NULL && ptr_ != default_value) { Destroy(default_value, arena); } @@ -147,7 +155,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { arena->Own(value); } } else { - ptr_ = const_cast< ::std::string* >(default_value); + ptr_ = const_cast< ::std::string*>(default_value); } } @@ -157,22 +165,23 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { // to implement unsafe_arena_set_allocated_ in generated classes. inline void UnsafeArenaSetAllocated(const ::std::string* default_value, ::std::string* value, - ::google::protobuf::Arena* /* arena */) { + Arena* /* arena */) { if (value != NULL) { ptr_ = value; } else { - ptr_ = const_cast< ::std::string* >(default_value); + ptr_ = const_cast< ::std::string*>(default_value); } } // Swaps internal pointers. Arena-safety semantics: this is guarded by the // logic in Swap()/UnsafeArenaSwap() at the message level, so this method is // 'unsafe' if called directly. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Swap(ArenaStringPtr* other) { + PROTOBUF_ALWAYS_INLINE void Swap(ArenaStringPtr* other) { std::swap(ptr_, other->ptr_); } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Swap( - ArenaStringPtr* other, const ::std::string* default_value, Arena* arena) { + PROTOBUF_ALWAYS_INLINE void Swap(ArenaStringPtr* other, + const ::std::string* default_value, + Arena* arena) { #ifndef NDEBUG // For debug builds, we swap the contents of the string, rather than the // string instances themselves. This invalidates previously taken const @@ -192,12 +201,13 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { this_ptr->swap(*other_ptr); #else std::swap(ptr_, other->ptr_); + (void)default_value; + (void)arena; #endif } // Frees storage (if not on an arena). - inline void Destroy(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { + inline void Destroy(const ::std::string* default_value, Arena* arena) { if (arena == NULL && ptr_ != default_value) { delete ptr_; } @@ -208,7 +218,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { // the user) will always be the empty string. Assumes that |default_value| // is an empty string. inline void ClearToEmpty(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { + Arena* /* arena */) { if (ptr_ == default_value) { // Already set to default (which is empty) -- do nothing. } else { @@ -218,18 +228,14 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { // Clears content, assuming that the current value is not the empty string // default. - inline void ClearNonDefaultToEmpty() { - ptr_->clear(); - } - inline void ClearNonDefaultToEmptyNoArena() { - ptr_->clear(); - } + inline void ClearNonDefaultToEmpty() { ptr_->clear(); } + inline void ClearNonDefaultToEmptyNoArena() { ptr_->clear(); } // Clears content, but keeps allocated string if arena != NULL, to avoid the // overhead of heap operations. After this returns, the content (as seen by // the user) will always be equal to |default_value|. inline void ClearToDefault(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { + Arena* /* arena */) { if (ptr_ == default_value) { // Already set to default -- do nothing. } else { @@ -248,7 +254,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { inline void UnsafeSetDefault(const ::std::string* default_value) { // Casting away 'const' is safe here: accessors ensure that ptr_ is only // returned as a const if it is equal to default_value. - ptr_ = const_cast< ::std::string* >(default_value); + ptr_ = const_cast< ::std::string*>(default_value); } // The 'NoArena' variants of methods below assume arena == NULL and are @@ -267,7 +273,6 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { } } -#if LANG_CXX11 void SetNoArena(const ::std::string* default_value, ::std::string&& value) { if (IsDefault(default_value)) { ptr_ = new ::std::string(std::move(value)); @@ -275,9 +280,9 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { *ptr_ = std::move(value); } } -#endif - void AssignWithDefault(const ::std::string* default_value, ArenaStringPtr value); + void AssignWithDefault(const ::std::string* default_value, + ArenaStringPtr value); inline const ::std::string& GetNoArena() const { return *ptr_; } @@ -300,11 +305,10 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { const ::std::string* default_value) { GOOGLE_DCHECK(!IsDefault(default_value)); ::std::string* released = ptr_; - ptr_ = const_cast< ::std::string* >(default_value); + ptr_ = const_cast< ::std::string*>(default_value); return released; } - inline void SetAllocatedNoArena(const ::std::string* default_value, ::std::string* value) { if (ptr_ != default_value) { @@ -313,7 +317,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { if (value != NULL) { ptr_ = value; } else { - ptr_ = const_cast< ::std::string* >(default_value); + ptr_ = const_cast< ::std::string*>(default_value); } } @@ -344,9 +348,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { // raw pointer from the shared parse routine (in the non-arenas case). The // parse routine does the string allocation in order to save code size in the // generated parsing code. - inline ::std::string** UnsafeRawStringPointer() { - return &ptr_; - } + inline ::std::string** UnsafeRawStringPointer() { return &ptr_; } inline bool IsDefault(const ::std::string* default_value) const { return ptr_ == default_value; @@ -364,14 +366,13 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { private: ::std::string* ptr_; - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void CreateInstance(::google::protobuf::Arena* arena, - const ::std::string* initial_value) { + PROTOBUF_NOINLINE + void CreateInstance(Arena* arena, const ::std::string* initial_value) { GOOGLE_DCHECK(initial_value != NULL); // uses "new ::std::string" when arena is nullptr - ptr_ = Arena::Create< ::std::string >(arena, *initial_value); + ptr_ = Arena::Create< ::std::string>(arena, *initial_value); } - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE + PROTOBUF_NOINLINE void CreateInstanceNoArena(const ::std::string* initial_value) { GOOGLE_DCHECK(initial_value != NULL); ptr_ = new ::std::string(*initial_value); @@ -381,13 +382,11 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr { } // namespace internal } // namespace protobuf - - namespace protobuf { namespace internal { -inline void ArenaStringPtr::AssignWithDefault(const ::std::string* default_value, - ArenaStringPtr value) { +inline void ArenaStringPtr::AssignWithDefault( + const ::std::string* default_value, ArenaStringPtr value) { const ::std::string* me = *UnsafeRawStringPointer(); const ::std::string* other = *value.UnsafeRawStringPointer(); // If the pointers are the same then do nothing. @@ -398,6 +397,9 @@ inline void ArenaStringPtr::AssignWithDefault(const ::std::string* default_value } // namespace internal } // namespace protobuf - } // namespace google + + +#include + #endif // GOOGLE_PROTOBUF_ARENASTRING_H__ diff --git a/third_party/protobuf-lite/google/protobuf/extension_set.h b/third_party/protobuf-lite/google/protobuf/extension_set.h index a1535baa..b30a9608 100644 --- a/third_party/protobuf-lite/google/protobuf/extension_set.h +++ b/third_party/protobuf-lite/google/protobuf/extension_set.h @@ -47,32 +47,40 @@ #include #include -#include +#include +#include +#include #include +#include -namespace google { +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif +namespace google { namespace protobuf { - class Arena; - class Descriptor; // descriptor.h - class FieldDescriptor; // descriptor.h - class DescriptorPool; // descriptor.h - class MessageLite; // message_lite.h - class Message; // message.h - class MessageFactory; // message.h - class UnknownFieldSet; // unknown_field_set.h - namespace io { - class CodedInputStream; // coded_stream.h - class CodedOutputStream; // coded_stream.h - } - namespace internal { - class FieldSkipper; // wire_format_lite.h - } -} +class Arena; +class Descriptor; // descriptor.h +class FieldDescriptor; // descriptor.h +class DescriptorPool; // descriptor.h +class MessageLite; // message_lite.h +class Message; // message.h +class MessageFactory; // message.h +class UnknownFieldSet; // unknown_field_set.h +namespace internal { +class FieldSkipper; // wire_format_lite.h +} // namespace internal +} // namespace protobuf +} // namespace google +namespace google { namespace protobuf { namespace internal { +class InternalMetadata; + // Used to store values of type WireFormatLite::FieldType without having to // #include wire_format_lite.h. Also, ensures that we use only one byte to // store these values, which is important to keep the layout of @@ -92,7 +100,9 @@ typedef bool EnumValidityFuncWithArg(const void* arg, int number); struct ExtensionInfo { inline ExtensionInfo() {} inline ExtensionInfo(FieldType type_param, bool isrepeated, bool ispacked) - : type(type_param), is_repeated(isrepeated), is_packed(ispacked), + : type(type_param), + is_repeated(isrepeated), + is_packed(ispacked), descriptor(NULL) {} FieldType type; @@ -104,9 +114,13 @@ struct ExtensionInfo { const void* arg; }; + struct MessageInfo { + const MessageLite* prototype; + }; + union { EnumValidityCheck enum_validity_check; - const MessageLite* message_prototype; + MessageInfo message_info; }; // The descriptor for this extension, if one exists and is known. May be @@ -117,7 +131,7 @@ struct ExtensionInfo { // Abstract interface for an object which looks up extension definitions. Used // when parsing. -class LIBPROTOBUF_EXPORT ExtensionFinder { +class PROTOBUF_EXPORT ExtensionFinder { public: virtual ~ExtensionFinder(); @@ -127,14 +141,14 @@ class LIBPROTOBUF_EXPORT ExtensionFinder { // Implementation of ExtensionFinder which finds extensions defined in .proto // files which have been compiled into the binary. -class LIBPROTOBUF_EXPORT GeneratedExtensionFinder : public ExtensionFinder { +class PROTOBUF_EXPORT GeneratedExtensionFinder : public ExtensionFinder { public: GeneratedExtensionFinder(const MessageLite* containing_type) : containing_type_(containing_type) {} - virtual ~GeneratedExtensionFinder() {} + ~GeneratedExtensionFinder() override {} // Returns true and fills in *output if found, otherwise returns false. - virtual bool Find(int number, ExtensionInfo* output); + bool Find(int number, ExtensionInfo* output) override; private: const MessageLite* containing_type_; @@ -157,10 +171,10 @@ class MessageSetFieldSkipper; // ExtensionSet. When parsing, if a tag number is encountered which is // inside one of the message type's extension ranges, the tag is passed // off to the ExtensionSet for parsing. Etc. -class LIBPROTOBUF_EXPORT ExtensionSet { +class PROTOBUF_EXPORT ExtensionSet { public: ExtensionSet(); - explicit ExtensionSet(::google::protobuf::Arena* arena); + explicit ExtensionSet(Arena* arena); ~ExtensionSet(); // These are called at startup by protocol-compiler-generated code to @@ -168,9 +182,9 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // to look up extensions for parsed field numbers. Note that dynamic parsing // does not use ParseField(); only protocol-compiler-generated parsing // methods do. - static void RegisterExtension(const MessageLite* containing_type, - int number, FieldType type, - bool is_repeated, bool is_packed); + static void RegisterExtension(const MessageLite* containing_type, int number, + FieldType type, bool is_repeated, + bool is_packed); static void RegisterEnumExtension(const MessageLite* containing_type, int number, FieldType type, bool is_repeated, bool is_packed, @@ -219,22 +233,23 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // - Strings provide Mutable() in addition to Set() accessors. bool Has(int number) const; - int ExtensionSize(int number) const; // Size of a repeated extension. - int NumExtensions() const; // The number of extensions + int ExtensionSize(int number) const; // Size of a repeated extension. + int NumExtensions() const; // The number of extensions FieldType ExtensionType(int number) const; void ClearExtension(int number); // singular fields ------------------------------------------------- - int32 GetInt32 (int number, int32 default_value) const; - int64 GetInt64 (int number, int64 default_value) const; + int32 GetInt32(int number, int32 default_value) const; + int64 GetInt64(int number, int64 default_value) const; uint32 GetUInt32(int number, uint32 default_value) const; uint64 GetUInt64(int number, uint64 default_value) const; - float GetFloat (int number, float default_value) const; + float GetFloat(int number, float default_value) const; double GetDouble(int number, double default_value) const; - bool GetBool (int number, bool default_value) const; - int GetEnum (int number, int default_value) const; - const string & GetString (int number, const string& default_value) const; + bool GetBool(int number, bool default_value) const; + int GetEnum(int number, int default_value) const; + const std::string& GetString(int number, + const std::string& default_value) const; const MessageLite& GetMessage(int number, const MessageLite& default_value) const; const MessageLite& GetMessage(int number, const Descriptor* message_type, @@ -244,19 +259,19 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // the extension lives in the same pool as the descriptor for the containing // type. #define desc const FieldDescriptor* descriptor // avoid line wrapping - void SetInt32 (int number, FieldType type, int32 value, desc); - void SetInt64 (int number, FieldType type, int64 value, desc); + void SetInt32(int number, FieldType type, int32 value, desc); + void SetInt64(int number, FieldType type, int64 value, desc); void SetUInt32(int number, FieldType type, uint32 value, desc); void SetUInt64(int number, FieldType type, uint64 value, desc); - void SetFloat (int number, FieldType type, float value, desc); + void SetFloat(int number, FieldType type, float value, desc); void SetDouble(int number, FieldType type, double value, desc); - void SetBool (int number, FieldType type, bool value, desc); - void SetEnum (int number, FieldType type, int value, desc); - void SetString(int number, FieldType type, const string& value, desc); - string * MutableString (int number, FieldType type, desc); + void SetBool(int number, FieldType type, bool value, desc); + void SetEnum(int number, FieldType type, int value, desc); + void SetString(int number, FieldType type, std::string value, desc); + std::string* MutableString(int number, FieldType type, desc); MessageLite* MutableMessage(int number, FieldType type, const MessageLite& prototype, desc); - MessageLite* MutableMessage(const FieldDescriptor* decsriptor, + MessageLite* MutableMessage(const FieldDescriptor* descriptor, MessageFactory* factory); // Adds the given message to the ExtensionSet, taking ownership of the // message object. Existing message with the same number will be deleted. @@ -268,15 +283,15 @@ class LIBPROTOBUF_EXPORT ExtensionSet { const FieldDescriptor* descriptor, MessageLite* message); MessageLite* ReleaseMessage(int number, const MessageLite& prototype); - MessageLite* UnsafeArenaReleaseMessage( - int number, const MessageLite& prototype); + MessageLite* UnsafeArenaReleaseMessage(int number, + const MessageLite& prototype); MessageLite* ReleaseMessage(const FieldDescriptor* descriptor, MessageFactory* factory); MessageLite* UnsafeArenaReleaseMessage(const FieldDescriptor* descriptor, MessageFactory* factory); #undef desc - ::google::protobuf::Arena* GetArenaNoVirtual() const { return arena_; } + Arena* GetArena() const { return arena_; } // repeated fields ------------------------------------------------- @@ -287,8 +302,8 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // Fetches a mutable version of a RepeatedField extension by number, // instantiating one if none exists. Similar to above, user should not use // this directly; it underlies MutableRepeatedExtension(). - void* MutableRawRepeatedField(int number, FieldType field_type, - bool packed, const FieldDescriptor* desc); + void* MutableRawRepeatedField(int number, FieldType field_type, bool packed, + const FieldDescriptor* desc); // This is an overload of MutableRawRepeatedField to maintain compatibility // with old code using a previous API. This version of @@ -296,40 +311,40 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // (E.g.: borg/clients/internal/proto1/proto2_reflection.cc.) void* MutableRawRepeatedField(int number); - int32 GetRepeatedInt32 (int number, int index) const; - int64 GetRepeatedInt64 (int number, int index) const; + int32 GetRepeatedInt32(int number, int index) const; + int64 GetRepeatedInt64(int number, int index) const; uint32 GetRepeatedUInt32(int number, int index) const; uint64 GetRepeatedUInt64(int number, int index) const; - float GetRepeatedFloat (int number, int index) const; + float GetRepeatedFloat(int number, int index) const; double GetRepeatedDouble(int number, int index) const; - bool GetRepeatedBool (int number, int index) const; - int GetRepeatedEnum (int number, int index) const; - const string & GetRepeatedString (int number, int index) const; + bool GetRepeatedBool(int number, int index) const; + int GetRepeatedEnum(int number, int index) const; + const std::string& GetRepeatedString(int number, int index) const; const MessageLite& GetRepeatedMessage(int number, int index) const; - void SetRepeatedInt32 (int number, int index, int32 value); - void SetRepeatedInt64 (int number, int index, int64 value); + void SetRepeatedInt32(int number, int index, int32 value); + void SetRepeatedInt64(int number, int index, int64 value); void SetRepeatedUInt32(int number, int index, uint32 value); void SetRepeatedUInt64(int number, int index, uint64 value); - void SetRepeatedFloat (int number, int index, float value); + void SetRepeatedFloat(int number, int index, float value); void SetRepeatedDouble(int number, int index, double value); - void SetRepeatedBool (int number, int index, bool value); - void SetRepeatedEnum (int number, int index, int value); - void SetRepeatedString(int number, int index, const string& value); - string * MutableRepeatedString (int number, int index); + void SetRepeatedBool(int number, int index, bool value); + void SetRepeatedEnum(int number, int index, int value); + void SetRepeatedString(int number, int index, std::string value); + std::string* MutableRepeatedString(int number, int index); MessageLite* MutableRepeatedMessage(int number, int index); #define desc const FieldDescriptor* descriptor // avoid line wrapping - void AddInt32 (int number, FieldType type, bool packed, int32 value, desc); - void AddInt64 (int number, FieldType type, bool packed, int64 value, desc); + void AddInt32(int number, FieldType type, bool packed, int32 value, desc); + void AddInt64(int number, FieldType type, bool packed, int64 value, desc); void AddUInt32(int number, FieldType type, bool packed, uint32 value, desc); void AddUInt64(int number, FieldType type, bool packed, uint64 value, desc); - void AddFloat (int number, FieldType type, bool packed, float value, desc); + void AddFloat(int number, FieldType type, bool packed, float value, desc); void AddDouble(int number, FieldType type, bool packed, double value, desc); - void AddBool (int number, FieldType type, bool packed, bool value, desc); - void AddEnum (int number, FieldType type, bool packed, int value, desc); - void AddString(int number, FieldType type, const string& value, desc); - string * AddString (int number, FieldType type, desc); + void AddBool(int number, FieldType type, bool packed, bool value, desc); + void AddEnum(int number, FieldType type, bool packed, int value, desc); + void AddString(int number, FieldType type, std::string value, desc); + std::string* AddString(int number, FieldType type, desc); MessageLite* AddMessage(int number, FieldType type, const MessageLite& prototype, desc); MessageLite* AddMessage(const FieldDescriptor* descriptor, @@ -378,8 +393,52 @@ class LIBPROTOBUF_EXPORT ExtensionSet { const MessageLite* containing_type, io::CodedOutputStream* unknown_fields); + // Lite parser + const char* ParseField(uint64 tag, const char* ptr, + const MessageLite* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx); + // Full parser + const char* ParseField(uint64 tag, const char* ptr, + const Message* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx); + template + const char* ParseMessageSet(const char* ptr, const Msg* containing_type, + InternalMetadata* metadata, + internal::ParseContext* ctx) { + struct MessageSetItem { + const char* _InternalParse(const char* ptr, ParseContext* ctx) { + return me->ParseMessageSetItem(ptr, containing_type, metadata, ctx); + } + ExtensionSet* me; + const Msg* containing_type; + InternalMetadata* metadata; + } item{this, containing_type, metadata}; + while (!ctx->Done(&ptr)) { + uint32 tag; + ptr = ReadTag(ptr, &tag); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + if (tag == WireFormatLite::kMessageSetItemStartTag) { + ptr = ctx->ParseGroup(&item, ptr, tag); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + } else { + if (tag == 0 || (tag & 7) == 4) { + ctx->SetLastTag(tag); + return ptr; + } + ptr = ParseField(tag, ptr, containing_type, metadata, ctx); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + } + } + return ptr; + } + // Parse an entire message in MessageSet format. Such messages have no // fields, only extensions. + bool ParseMessageSetLite(io::CodedInputStream* input, + ExtensionFinder* extension_finder, + FieldSkipper* field_skipper); bool ParseMessageSet(io::CodedInputStream* input, ExtensionFinder* extension_finder, MessageSetFieldSkipper* field_skipper); @@ -387,7 +446,8 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // Specific versions for lite or full messages (constructs the appropriate // FieldSkipper automatically). bool ParseMessageSet(io::CodedInputStream* input, - const MessageLite* containing_type); + const MessageLite* containing_type, + std::string* unknown_fields); bool ParseMessageSet(io::CodedInputStream* input, const Message* containing_type, UnknownFieldSet* unknown_fields); @@ -396,24 +456,28 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // [start_field_number, end_field_number) // to the output stream, using the cached sizes computed when ByteSize() was // last called. Note that the range bounds are inclusive-exclusive. - void SerializeWithCachedSizes(int start_field_number, - int end_field_number, - io::CodedOutputStream* output) const; + void SerializeWithCachedSizes(int start_field_number, int end_field_number, + io::CodedOutputStream* output) const { + output->SetCur(_InternalSerialize(start_field_number, end_field_number, + output->Cur(), output->EpsCopy())); + } // Same as SerializeWithCachedSizes, but without any bounds checking. // The caller must ensure that target has sufficient capacity for the // serialized extensions. // // Returns a pointer past the last written byte. - uint8* InternalSerializeWithCachedSizesToArray(int start_field_number, - int end_field_number, - bool deterministic, - uint8* target) const; + uint8* _InternalSerialize(int start_field_number, int end_field_number, + uint8* target, + io::EpsCopyOutputStream* stream) const; // Like above but serializes in MessageSet format. - void SerializeMessageSetWithCachedSizes(io::CodedOutputStream* output) const; - uint8* InternalSerializeMessageSetWithCachedSizesToArray(bool deterministic, - uint8* target) const; + void SerializeMessageSetWithCachedSizes(io::CodedOutputStream* output) const { + output->SetCur(InternalSerializeMessageSetWithCachedSizesToArray( + output->Cur(), output->EpsCopy())); + } + uint8* InternalSerializeMessageSetWithCachedSizesToArray( + uint8* target, io::EpsCopyOutputStream* stream) const; // For backward-compatibility, versions of two of the above methods that // serialize deterministically iff SetDefaultSerializationDeterministic() @@ -446,29 +510,26 @@ class LIBPROTOBUF_EXPORT ExtensionSet { int SpaceUsedExcludingSelf() const; private: - // Interface of a lazily parsed singular message extension. - class LIBPROTOBUF_EXPORT LazyMessageExtension { + class PROTOBUF_EXPORT LazyMessageExtension { public: LazyMessageExtension() {} virtual ~LazyMessageExtension() {} - virtual LazyMessageExtension* New(::google::protobuf::Arena* arena) const = 0; + virtual LazyMessageExtension* New(Arena* arena) const = 0; virtual const MessageLite& GetMessage( const MessageLite& prototype) const = 0; virtual MessageLite* MutableMessage(const MessageLite& prototype) = 0; - virtual void SetAllocatedMessage(MessageLite *message) = 0; - virtual void UnsafeArenaSetAllocatedMessage(MessageLite *message) = 0; + virtual void SetAllocatedMessage(MessageLite* message) = 0; + virtual void UnsafeArenaSetAllocatedMessage(MessageLite* message) = 0; virtual MessageLite* ReleaseMessage(const MessageLite& prototype) = 0; virtual MessageLite* UnsafeArenaReleaseMessage( const MessageLite& prototype) = 0; virtual bool IsInitialized() const = 0; - PROTOBUF_RUNTIME_DEPRECATED("Please use ByteSizeLong() instead") - virtual int ByteSize() const { - return internal::ToIntSize(ByteSizeLong()); - } + PROTOBUF_DEPRECATED_MSG("Please use ByteSizeLong() instead") + virtual int ByteSize() const { return internal::ToIntSize(ByteSizeLong()); } virtual size_t ByteSizeLong() const = 0; virtual size_t SpaceUsedLong() const = 0; @@ -477,15 +538,9 @@ class LIBPROTOBUF_EXPORT ExtensionSet { virtual bool ReadMessage(const MessageLite& prototype, io::CodedInputStream* input) = 0; - virtual void WriteMessage(int number, - io::CodedOutputStream* output) const = 0; - virtual uint8* WriteMessageToArray(int number, uint8* target) const = 0; - virtual uint8* InternalWriteMessageToArray(int number, bool, - uint8* target) const { - // TODO(gpike): make this pure virtual. This is a placeholder because we - // need to update third_party/upb, for example. - return WriteMessageToArray(number, target); - } + virtual const char* _InternalParse(const char* ptr, ParseContext* ctx) = 0; + virtual uint8* WriteMessageToArray( + int number, uint8* target, io::EpsCopyOutputStream* stream) const = 0; private: virtual void UnusedKeyMethod(); // Dummy key method to avoid weak vtable. @@ -496,27 +551,27 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // The order of these fields packs Extension into 24 bytes when using 8 // byte alignment. Consider this when adding or removing fields here. union { - int32 int32_value; - int64 int64_value; - uint32 uint32_value; - uint64 uint64_value; - float float_value; - double double_value; - bool bool_value; - int enum_value; - string* string_value; - MessageLite* message_value; + int32 int32_value; + int64 int64_value; + uint32 uint32_value; + uint64 uint64_value; + float float_value; + double double_value; + bool bool_value; + int enum_value; + std::string* string_value; + MessageLite* message_value; LazyMessageExtension* lazymessage_value; - RepeatedField * repeated_int32_value; - RepeatedField * repeated_int64_value; - RepeatedField * repeated_uint32_value; - RepeatedField * repeated_uint64_value; - RepeatedField * repeated_float_value; - RepeatedField * repeated_double_value; - RepeatedField * repeated_bool_value; - RepeatedField * repeated_enum_value; - RepeatedPtrField* repeated_string_value; + RepeatedField* repeated_int32_value; + RepeatedField* repeated_int64_value; + RepeatedField* repeated_uint32_value; + RepeatedField* repeated_uint64_value; + RepeatedField* repeated_float_value; + RepeatedField* repeated_double_value; + RepeatedField* repeated_bool_value; + RepeatedField* repeated_enum_value; + RepeatedPtrField* repeated_string_value; RepeatedPtrField* repeated_message_value; }; @@ -552,20 +607,10 @@ class LIBPROTOBUF_EXPORT ExtensionSet { const FieldDescriptor* descriptor; // Some helper methods for operations on a single Extension. - void SerializeFieldWithCachedSizes( - int number, - io::CodedOutputStream* output) const; uint8* InternalSerializeFieldWithCachedSizesToArray( - int number, - bool deterministic, - uint8* target) const; - void SerializeMessageSetItemWithCachedSizes( - int number, - io::CodedOutputStream* output) const; + int number, uint8* target, io::EpsCopyOutputStream* stream) const; uint8* InternalSerializeMessageSetItemWithCachedSizesToArray( - int number, - bool deterministic, - uint8* target) const; + int number, uint8* target, io::EpsCopyOutputStream* stream) const; size_t ByteSize(int number) const; size_t MessageSetItemByteSize(int number) const; void Clear(); @@ -625,7 +670,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet { void Erase(int key); size_t Size() const { - return GOOGLE_PREDICT_FALSE(is_large()) ? map_.large->size() : flat_size_; + return PROTOBUF_PREDICT_FALSE(is_large()) ? map_.large->size() : flat_size_; } // Similar to std::for_each. @@ -641,7 +686,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // Applies a functor to the pairs in sorted order. template KeyValueFunctor ForEach(KeyValueFunctor func) { - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { return ForEach(map_.large->begin(), map_.large->end(), std::move(func)); } return ForEach(flat_begin(), flat_end(), std::move(func)); @@ -650,7 +695,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // Applies a functor to the pairs in sorted order. template KeyValueFunctor ForEach(KeyValueFunctor func) const { - if (GOOGLE_PREDICT_FALSE(is_large())) { + if (PROTOBUF_PREDICT_FALSE(is_large())) { return ForEach(map_.large->begin(), map_.large->end(), std::move(func)); } return ForEach(flat_begin(), flat_end(), std::move(func)); @@ -680,8 +725,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // positioned immediately after the wire tag. This method is called in // ParseField() after field number and was_packed_on_wire is extracted from // the wire tag and ExtensionInfo is found by the field number. - bool ParseFieldWithExtensionInfo(int field_number, - bool was_packed_on_wire, + bool ParseFieldWithExtensionInfo(int field_number, bool was_packed_on_wire, const ExtensionInfo& extension, io::CodedInputStream* input, FieldSkipper* field_skipper); @@ -702,12 +746,63 @@ class LIBPROTOBUF_EXPORT ExtensionSet { // it does not exist. Extension* MaybeNewRepeatedExtension(const FieldDescriptor* descriptor); + // Parse a single MessageSet item -- called just after the item group start + // tag has been read. + bool ParseMessageSetItemLite(io::CodedInputStream* input, + ExtensionFinder* extension_finder, + FieldSkipper* field_skipper); // Parse a single MessageSet item -- called just after the item group start // tag has been read. bool ParseMessageSetItem(io::CodedInputStream* input, ExtensionFinder* extension_finder, MessageSetFieldSkipper* field_skipper); + bool FindExtension(int wire_type, uint32 field, + const MessageLite* containing_type, + const internal::ParseContext* /*ctx*/, + ExtensionInfo* extension, bool* was_packed_on_wire) { + GeneratedExtensionFinder finder(containing_type); + return FindExtensionInfoFromFieldNumber(wire_type, field, &finder, + extension, was_packed_on_wire); + } + inline bool FindExtension(int wire_type, uint32 field, + const Message* containing_type, + const internal::ParseContext* ctx, + ExtensionInfo* extension, bool* was_packed_on_wire); + // Used for MessageSet only + const char* ParseFieldMaybeLazily(uint64 tag, const char* ptr, + const MessageLite* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx) { + // Lite MessageSet doesn't implement lazy. + return ParseField(tag, ptr, containing_type, metadata, ctx); + } + const char* ParseFieldMaybeLazily(uint64 tag, const char* ptr, + const Message* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx); + const char* ParseMessageSetItem(const char* ptr, + const MessageLite* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx); + const char* ParseMessageSetItem(const char* ptr, + const Message* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx); + + // Implemented in extension_set_inl.h to keep code out of the header file. + template + const char* ParseFieldWithExtensionInfo(int number, bool was_packed_on_wire, + const ExtensionInfo& info, + internal::InternalMetadata* metadata, + const char* ptr, + internal::ParseContext* ctx); + template + const char* ParseMessageSetItemTmpl(const char* ptr, + const Msg* containing_type, + internal::InternalMetadata* metadata, + internal::ParseContext* ctx); + // Hack: RepeatedPtrFieldBase declares ExtensionSet as a friend. This // friendship should automatically extend to ExtensionSet::Extension, but // unfortunately some older compilers (e.g. GCC 3.4.4) do not implement this @@ -735,7 +830,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet { return map_.flat + flat_size_; } - ::google::protobuf::Arena* arena_; + Arena* arena_; // Manual memory-management: // map_.flat is an allocated array of flat_capacity_ elements. @@ -750,25 +845,26 @@ class LIBPROTOBUF_EXPORT ExtensionSet { LargeMap* large; } map_; + static void DeleteFlatMap(const KeyValue* flat, uint16 flat_capacity); + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ExtensionSet); }; // These are just for convenience... inline void ExtensionSet::SetString(int number, FieldType type, - const string& value, + std::string value, const FieldDescriptor* descriptor) { - MutableString(number, type, descriptor)->assign(value); + MutableString(number, type, descriptor)->assign(std::move(value)); } inline void ExtensionSet::SetRepeatedString(int number, int index, - const string& value) { - MutableRepeatedString(number, index)->assign(value); + std::string value) { + MutableRepeatedString(number, index)->assign(std::move(value)); } inline void ExtensionSet::AddString(int number, FieldType type, - const string& value, + std::string value, const FieldDescriptor* descriptor) { - AddString(number, type, descriptor)->assign(value); + AddString(number, type, descriptor)->assign(std::move(value)); } - // =================================================================== // Glue for generated extension accessors @@ -841,8 +937,8 @@ class PrimitiveTypeTraits { static inline ConstType Get(int number, const ExtensionSet& set, ConstType default_value); - static inline void Set(int number, FieldType field_type, - ConstType value, ExtensionSet* set); + static inline void Set(int number, FieldType field_type, ConstType value, + ExtensionSet* set); template static void Register(int number, FieldType type, bool is_packed) { ExtensionSet::RegisterExtension(&ExtendeeT::default_instance(), number, @@ -861,14 +957,15 @@ class RepeatedPrimitiveTypeTraits { static inline Type Get(int number, const ExtensionSet& set, int index); static inline void Set(int number, int index, Type value, ExtensionSet* set); - static inline void Add(int number, FieldType field_type, - bool is_packed, Type value, ExtensionSet* set); + static inline void Add(int number, FieldType field_type, bool is_packed, + Type value, ExtensionSet* set); - static inline const RepeatedField& - GetRepeated(int number, const ExtensionSet& set); - static inline RepeatedField* - MutableRepeated(int number, FieldType field_type, - bool is_packed, ExtensionSet* set); + static inline const RepeatedField& GetRepeated( + int number, const ExtensionSet& set); + static inline RepeatedField* MutableRepeated(int number, + FieldType field_type, + bool is_packed, + ExtensionSet* set); static const RepeatedFieldType* GetDefaultRepeatedField(); template @@ -878,11 +975,10 @@ class RepeatedPrimitiveTypeTraits { } }; -LIBPROTOBUF_EXPORT extern ProtobufOnceType repeated_primitive_generic_type_traits_once_init_; - -class LIBPROTOBUF_EXPORT RepeatedPrimitiveDefaults { +class PROTOBUF_EXPORT RepeatedPrimitiveDefaults { private: - template friend class RepeatedPrimitiveTypeTraits; + template + friend class RepeatedPrimitiveTypeTraits; static const RepeatedPrimitiveDefaults* default_instance(); RepeatedField default_repeated_field_int32_; RepeatedField default_repeated_field_int64_; @@ -893,57 +989,62 @@ class LIBPROTOBUF_EXPORT RepeatedPrimitiveDefaults { RepeatedField default_repeated_field_bool_; }; -#define PROTOBUF_DEFINE_PRIMITIVE_TYPE(TYPE, METHOD) \ -template<> inline TYPE PrimitiveTypeTraits::Get( \ - int number, const ExtensionSet& set, TYPE default_value) { \ - return set.Get##METHOD(number, default_value); \ -} \ -template<> inline void PrimitiveTypeTraits::Set( \ - int number, FieldType field_type, TYPE value, ExtensionSet* set) { \ - set->Set##METHOD(number, field_type, value, NULL); \ -} \ - \ -template<> inline TYPE RepeatedPrimitiveTypeTraits::Get( \ - int number, const ExtensionSet& set, int index) { \ - return set.GetRepeated##METHOD(number, index); \ -} \ -template<> inline void RepeatedPrimitiveTypeTraits::Set( \ - int number, int index, TYPE value, ExtensionSet* set) { \ - set->SetRepeated##METHOD(number, index, value); \ -} \ -template<> inline void RepeatedPrimitiveTypeTraits::Add( \ - int number, FieldType field_type, bool is_packed, \ - TYPE value, ExtensionSet* set) { \ - set->Add##METHOD(number, field_type, is_packed, value, NULL); \ -} \ -template<> inline const RepeatedField* \ - RepeatedPrimitiveTypeTraits::GetDefaultRepeatedField() { \ - return &RepeatedPrimitiveDefaults::default_instance() \ - ->default_repeated_field_##TYPE##_; \ -} \ -template<> inline const RepeatedField& \ - RepeatedPrimitiveTypeTraits::GetRepeated(int number, \ - const ExtensionSet& set) { \ - return *reinterpret_cast*>( \ - set.GetRawRepeatedField( \ - number, GetDefaultRepeatedField())); \ -} \ -template<> inline RepeatedField* \ - RepeatedPrimitiveTypeTraits::MutableRepeated(int number, \ - FieldType field_type, \ - bool is_packed, \ - ExtensionSet* set) { \ - return reinterpret_cast*>( \ - set->MutableRawRepeatedField(number, field_type, is_packed, NULL)); \ -} +#define PROTOBUF_DEFINE_PRIMITIVE_TYPE(TYPE, METHOD) \ + template <> \ + inline TYPE PrimitiveTypeTraits::Get( \ + int number, const ExtensionSet& set, TYPE default_value) { \ + return set.Get##METHOD(number, default_value); \ + } \ + template <> \ + inline void PrimitiveTypeTraits::Set(int number, FieldType field_type, \ + TYPE value, ExtensionSet* set) { \ + set->Set##METHOD(number, field_type, value, NULL); \ + } \ + \ + template <> \ + inline TYPE RepeatedPrimitiveTypeTraits::Get( \ + int number, const ExtensionSet& set, int index) { \ + return set.GetRepeated##METHOD(number, index); \ + } \ + template <> \ + inline void RepeatedPrimitiveTypeTraits::Set( \ + int number, int index, TYPE value, ExtensionSet* set) { \ + set->SetRepeated##METHOD(number, index, value); \ + } \ + template <> \ + inline void RepeatedPrimitiveTypeTraits::Add( \ + int number, FieldType field_type, bool is_packed, TYPE value, \ + ExtensionSet* set) { \ + set->Add##METHOD(number, field_type, is_packed, value, NULL); \ + } \ + template <> \ + inline const RepeatedField* \ + RepeatedPrimitiveTypeTraits::GetDefaultRepeatedField() { \ + return &RepeatedPrimitiveDefaults::default_instance() \ + ->default_repeated_field_##TYPE##_; \ + } \ + template <> \ + inline const RepeatedField& \ + RepeatedPrimitiveTypeTraits::GetRepeated(int number, \ + const ExtensionSet& set) { \ + return *reinterpret_cast*>( \ + set.GetRawRepeatedField(number, GetDefaultRepeatedField())); \ + } \ + template <> \ + inline RepeatedField* \ + RepeatedPrimitiveTypeTraits::MutableRepeated( \ + int number, FieldType field_type, bool is_packed, ExtensionSet* set) { \ + return reinterpret_cast*>( \ + set->MutableRawRepeatedField(number, field_type, is_packed, NULL)); \ + } -PROTOBUF_DEFINE_PRIMITIVE_TYPE( int32, Int32) -PROTOBUF_DEFINE_PRIMITIVE_TYPE( int64, Int64) +PROTOBUF_DEFINE_PRIMITIVE_TYPE(int32, Int32) +PROTOBUF_DEFINE_PRIMITIVE_TYPE(int64, Int64) PROTOBUF_DEFINE_PRIMITIVE_TYPE(uint32, UInt32) PROTOBUF_DEFINE_PRIMITIVE_TYPE(uint64, UInt64) -PROTOBUF_DEFINE_PRIMITIVE_TYPE( float, Float) +PROTOBUF_DEFINE_PRIMITIVE_TYPE(float, Float) PROTOBUF_DEFINE_PRIMITIVE_TYPE(double, Double) -PROTOBUF_DEFINE_PRIMITIVE_TYPE( bool, Bool) +PROTOBUF_DEFINE_PRIMITIVE_TYPE(bool, Bool) #undef PROTOBUF_DEFINE_PRIMITIVE_TYPE @@ -951,22 +1052,22 @@ PROTOBUF_DEFINE_PRIMITIVE_TYPE( bool, Bool) // StringTypeTraits // Strings support both Set() and Mutable(). -class LIBPROTOBUF_EXPORT StringTypeTraits { +class PROTOBUF_EXPORT StringTypeTraits { public: - typedef const string& ConstType; - typedef string* MutableType; + typedef const std::string& ConstType; + typedef std::string* MutableType; typedef StringTypeTraits Singular; - static inline const string& Get(int number, const ExtensionSet& set, - ConstType default_value) { + static inline const std::string& Get(int number, const ExtensionSet& set, + ConstType default_value) { return set.GetString(number, default_value); } static inline void Set(int number, FieldType field_type, - const string& value, ExtensionSet* set) { + const std::string& value, ExtensionSet* set) { set->SetString(number, field_type, value, NULL); } - static inline string* Mutable(int number, FieldType field_type, - ExtensionSet* set) { + static inline std::string* Mutable(int number, FieldType field_type, + ExtensionSet* set) { return set->MutableString(number, field_type, NULL); } template @@ -976,46 +1077,43 @@ class LIBPROTOBUF_EXPORT StringTypeTraits { } }; -class LIBPROTOBUF_EXPORT RepeatedStringTypeTraits { +class PROTOBUF_EXPORT RepeatedStringTypeTraits { public: - typedef const string& ConstType; - typedef string* MutableType; + typedef const std::string& ConstType; + typedef std::string* MutableType; typedef RepeatedStringTypeTraits Repeated; - typedef RepeatedPtrField RepeatedFieldType; + typedef RepeatedPtrField RepeatedFieldType; - static inline const string& Get(int number, const ExtensionSet& set, - int index) { + static inline const std::string& Get(int number, const ExtensionSet& set, + int index) { return set.GetRepeatedString(number, index); } - static inline void Set(int number, int index, - const string& value, ExtensionSet* set) { + static inline void Set(int number, int index, const std::string& value, + ExtensionSet* set) { set->SetRepeatedString(number, index, value); } - static inline string* Mutable(int number, int index, ExtensionSet* set) { + static inline std::string* Mutable(int number, int index, ExtensionSet* set) { return set->MutableRepeatedString(number, index); } - static inline void Add(int number, FieldType field_type, - bool /*is_packed*/, const string& value, - ExtensionSet* set) { + static inline void Add(int number, FieldType field_type, bool /*is_packed*/, + const std::string& value, ExtensionSet* set) { set->AddString(number, field_type, value, NULL); } - static inline string* Add(int number, FieldType field_type, - ExtensionSet* set) { + static inline std::string* Add(int number, FieldType field_type, + ExtensionSet* set) { return set->AddString(number, field_type, NULL); } - static inline const RepeatedPtrField& - GetRepeated(int number, const ExtensionSet& set) { - return *reinterpret_cast*>( + static inline const RepeatedPtrField& GetRepeated( + int number, const ExtensionSet& set) { + return *reinterpret_cast*>( set.GetRawRepeatedField(number, GetDefaultRepeatedField())); } - static inline RepeatedPtrField* - MutableRepeated(int number, FieldType field_type, - bool is_packed, ExtensionSet* set) { - return reinterpret_cast*>( - set->MutableRawRepeatedField(number, field_type, - is_packed, NULL)); + static inline RepeatedPtrField* MutableRepeated( + int number, FieldType field_type, bool is_packed, ExtensionSet* set) { + return reinterpret_cast*>( + set->MutableRawRepeatedField(number, field_type, is_packed, NULL)); } static const RepeatedFieldType* GetDefaultRepeatedField(); @@ -1047,8 +1145,8 @@ class EnumTypeTraits { ConstType default_value) { return static_cast(set.GetEnum(number, default_value)); } - static inline void Set(int number, FieldType field_type, - ConstType value, ExtensionSet* set) { + static inline void Set(int number, FieldType field_type, ConstType value, + ExtensionSet* set) { GOOGLE_DCHECK(IsValid(value)); set->SetEnum(number, field_type, value, NULL); } @@ -1071,19 +1169,18 @@ class RepeatedEnumTypeTraits { static inline ConstType Get(int number, const ExtensionSet& set, int index) { return static_cast(set.GetRepeatedEnum(number, index)); } - static inline void Set(int number, int index, - ConstType value, ExtensionSet* set) { + static inline void Set(int number, int index, ConstType value, + ExtensionSet* set) { GOOGLE_DCHECK(IsValid(value)); set->SetRepeatedEnum(number, index, value); } - static inline void Add(int number, FieldType field_type, - bool is_packed, ConstType value, ExtensionSet* set) { + static inline void Add(int number, FieldType field_type, bool is_packed, + ConstType value, ExtensionSet* set) { GOOGLE_DCHECK(IsValid(value)); set->AddEnum(number, field_type, is_packed, value, NULL); } - static inline const RepeatedField& GetRepeated(int number, - const ExtensionSet& - set) { + static inline const RepeatedField& GetRepeated( + int number, const ExtensionSet& set) { // Hack: the `Extension` struct stores a RepeatedField for enums. // RepeatedField cannot implicitly convert to RepeatedField // so we need to do some casting magic. See message.h for similar @@ -1132,13 +1229,12 @@ class MessageTypeTraits { static inline ConstType Get(int number, const ExtensionSet& set, ConstType default_value) { - return static_cast( - set.GetMessage(number, default_value)); + return static_cast(set.GetMessage(number, default_value)); } static inline MutableType Mutable(int number, FieldType field_type, ExtensionSet* set) { - return static_cast( - set->MutableMessage(number, field_type, Type::default_instance(), NULL)); + return static_cast(set->MutableMessage( + number, field_type, Type::default_instance(), NULL)); } static inline void SetAllocated(int number, FieldType field_type, MutableType message, ExtensionSet* set) { @@ -1151,14 +1247,14 @@ class MessageTypeTraits { } static inline MutableType Release(int number, FieldType /* field_type */, ExtensionSet* set) { - return static_cast(set->ReleaseMessage( - number, Type::default_instance())); + return static_cast( + set->ReleaseMessage(number, Type::default_instance())); } static inline MutableType UnsafeArenaRelease(int number, FieldType /* field_type */, ExtensionSet* set) { - return static_cast(set->UnsafeArenaReleaseMessage( - number, Type::default_instance())); + return static_cast( + set->UnsafeArenaReleaseMessage(number, Type::default_instance())); } template static void Register(int number, FieldType type, bool is_packed) { @@ -1191,9 +1287,8 @@ class RepeatedMessageTypeTraits { return static_cast( set->AddMessage(number, field_type, Type::default_instance(), NULL)); } - static inline const RepeatedPtrField& GetRepeated(int number, - const ExtensionSet& - set) { + static inline const RepeatedPtrField& GetRepeated( + int number, const ExtensionSet& set) { // See notes above in RepeatedEnumTypeTraits::GetRepeated(): same // casting hack applies here, because a RepeatedPtrField // cannot naturally become a RepeatedPtrType even though Type is @@ -1219,9 +1314,9 @@ class RepeatedMessageTypeTraits { } }; -template inline - const typename RepeatedMessageTypeTraits::RepeatedFieldType* - RepeatedMessageTypeTraits::GetDefaultRepeatedField() { +template +inline const typename RepeatedMessageTypeTraits::RepeatedFieldType* +RepeatedMessageTypeTraits::GetDefaultRepeatedField() { static auto instance = OnShutdownDelete(new RepeatedFieldType); return instance; } @@ -1232,19 +1327,19 @@ template inline // This is the type of actual extension objects. E.g. if you have: // extends Foo with optional int32 bar = 1234; // then "bar" will be defined in C++ as: -// ExtensionIdentifier, 1, false> bar(1234); +// ExtensionIdentifier, 5, false> bar(1234); // // Note that we could, in theory, supply the field number as a template // parameter, and thus make an instance of ExtensionIdentifier have no -// actual contents. However, if we did that, then using at extension +// actual contents. However, if we did that, then using an extension // identifier would not necessarily cause the compiler to output any sort // of reference to any symbol defined in the extension's .pb.o file. Some // linkers will actually drop object files that are not explicitly referenced, // but that would be bad because it would cause this extension to not be // registered at static initialization, and therefore using it would crash. -template +template class ExtensionIdentifier { public: typedef TypeTraitsType TypeTraits; @@ -1281,182 +1376,213 @@ class ExtensionIdentifier { // // For similar reason, we use "_field_type" and "_is_packed" as parameter names // below, so that "field_type" and "is_packed" can be used as field names. -#define GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(CLASSNAME) \ +#define GOOGLE_PROTOBUF_EXTENSION_ACCESSORS(CLASSNAME) \ /* Has, Size, Clear */ \ template \ inline bool HasExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) const { \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) const { \ return _extensions_.Has(id.number()); \ } \ \ template \ inline void ClearExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ _extensions_.ClearExtension(id.number()); \ } \ \ template \ inline int ExtensionSize( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) const { \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) const { \ return _extensions_.ExtensionSize(id.number()); \ } \ \ /* Singular accessors */ \ template \ inline typename _proto_TypeTraits::Singular::ConstType GetExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) const { \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) const { \ return _proto_TypeTraits::Get(id.number(), _extensions_, \ id.default_value()); \ } \ \ template \ inline typename _proto_TypeTraits::Singular::MutableType MutableExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ return _proto_TypeTraits::Mutable(id.number(), _field_type, \ &_extensions_); \ } \ \ template \ inline void SetExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ typename _proto_TypeTraits::Singular::ConstType value) { \ _proto_TypeTraits::Set(id.number(), _field_type, value, &_extensions_); \ } \ \ template \ inline void SetAllocatedExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ typename _proto_TypeTraits::Singular::MutableType value) { \ - _proto_TypeTraits::SetAllocated(id.number(), _field_type, \ - value, &_extensions_); \ + _proto_TypeTraits::SetAllocated(id.number(), _field_type, value, \ + &_extensions_); \ } \ template \ inline void UnsafeArenaSetAllocatedExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ typename _proto_TypeTraits::Singular::MutableType value) { \ _proto_TypeTraits::UnsafeArenaSetAllocated(id.number(), _field_type, \ value, &_extensions_); \ } \ template \ inline typename _proto_TypeTraits::Singular::MutableType ReleaseExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ return _proto_TypeTraits::Release(id.number(), _field_type, \ &_extensions_); \ } \ template \ inline typename _proto_TypeTraits::Singular::MutableType \ - UnsafeArenaReleaseExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ + UnsafeArenaReleaseExtension( \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ return _proto_TypeTraits::UnsafeArenaRelease(id.number(), _field_type, \ &_extensions_); \ } \ \ /* Repeated accessors */ \ template \ inline typename _proto_TypeTraits::Repeated::ConstType GetExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ int index) const { \ return _proto_TypeTraits::Get(id.number(), _extensions_, index); \ } \ \ template \ inline typename _proto_TypeTraits::Repeated::MutableType MutableExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ int index) { \ return _proto_TypeTraits::Mutable(id.number(), index, &_extensions_); \ } \ \ template \ inline void SetExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ int index, typename _proto_TypeTraits::Repeated::ConstType value) { \ _proto_TypeTraits::Set(id.number(), index, value, &_extensions_); \ } \ \ template \ inline typename _proto_TypeTraits::Repeated::MutableType AddExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ return _proto_TypeTraits::Add(id.number(), _field_type, &_extensions_); \ } \ \ template \ inline void AddExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id, \ typename _proto_TypeTraits::Repeated::ConstType value) { \ - _proto_TypeTraits::Add(id.number(), _field_type, _is_packed, \ - value, &_extensions_); \ + _proto_TypeTraits::Add(id.number(), _field_type, _is_packed, value, \ + &_extensions_); \ } \ \ template \ inline const typename _proto_TypeTraits::Repeated::RepeatedFieldType& \ - GetRepeatedExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, \ - _is_packed>& id) const { \ + GetRepeatedExtension( \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) const { \ return _proto_TypeTraits::GetRepeated(id.number(), _extensions_); \ } \ \ template \ inline typename _proto_TypeTraits::Repeated::RepeatedFieldType* \ - MutableRepeatedExtension( \ - const ::google::protobuf::internal::ExtensionIdentifier< \ - CLASSNAME, _proto_TypeTraits, _field_type, \ - _is_packed>& id) { \ + MutableRepeatedExtension( \ + const ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< \ + CLASSNAME, _proto_TypeTraits, _field_type, _is_packed>& id) { \ return _proto_TypeTraits::MutableRepeated(id.number(), _field_type, \ _is_packed, &_extensions_); \ } } // namespace internal -} // namespace protobuf +// Call this function to ensure that this extensions's reflection is linked into +// the binary: +// +// google::protobuf::LinkExtensionReflection(Foo::my_extension); +// +// This will ensure that the following lookup will succeed: +// +// DescriptorPool::generated_pool()->FindExtensionByName("Foo.my_extension"); +// +// This is often relevant for parsing extensions in text mode. +// +// As a side-effect, it will also guarantee that anything else from the same +// .proto file will also be available for lookup in the generated pool. +// +// This function does not actually register the extension, so it does not need +// to be called before the lookup. However it does need to occur in a function +// that cannot be stripped from the binary (ie. it must be reachable from main). +// +// Best practice is to call this function as close as possible to where the +// reflection is actually needed. This function is very cheap to call, so you +// should not need to worry about its runtime overhead except in tight loops (on +// x86-64 it compiles into two "mov" instructions). +template +void LinkExtensionReflection( + const google::protobuf::internal::ExtensionIdentifier< + ExtendeeType, TypeTraitsType, field_type, is_packed>& extension) { + internal::StrongReference(extension); +} + +} // namespace protobuf } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_EXTENSION_SET_H__ diff --git a/third_party/protobuf-lite/google/protobuf/extension_set_inl.h b/third_party/protobuf-lite/google/protobuf/extension_set_inl.h new file mode 100644 index 00000000..9957f8ba --- /dev/null +++ b/third_party/protobuf-lite/google/protobuf/extension_set_inl.h @@ -0,0 +1,274 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__ +#define GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__ + +#include +#include +#include + +namespace google { +namespace protobuf { +namespace internal { + +template +const char* ExtensionSet::ParseFieldWithExtensionInfo( + int number, bool was_packed_on_wire, const ExtensionInfo& extension, + InternalMetadata* metadata, const char* ptr, internal::ParseContext* ctx) { + if (was_packed_on_wire) { + switch (extension.type) { +#define HANDLE_TYPE(UPPERCASE, CPP_CAMELCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: \ + return internal::Packed##CPP_CAMELCASE##Parser( \ + MutableRawRepeatedField(number, extension.type, extension.is_packed, \ + extension.descriptor), \ + ptr, ctx); + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(SINT32, SInt32); + HANDLE_TYPE(SINT64, SInt64); + HANDLE_TYPE(FIXED32, Fixed32); + HANDLE_TYPE(FIXED64, Fixed64); + HANDLE_TYPE(SFIXED32, SFixed32); + HANDLE_TYPE(SFIXED64, SFixed64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); +#undef HANDLE_TYPE + + case WireFormatLite::TYPE_ENUM: + return internal::PackedEnumParserArg( + MutableRawRepeatedField(number, extension.type, extension.is_packed, + extension.descriptor), + ptr, ctx, extension.enum_validity_check.func, + extension.enum_validity_check.arg, metadata, number); + case WireFormatLite::TYPE_STRING: + case WireFormatLite::TYPE_BYTES: + case WireFormatLite::TYPE_GROUP: + case WireFormatLite::TYPE_MESSAGE: + GOOGLE_LOG(FATAL) << "Non-primitive types can't be packed."; + break; + } + } else { + switch (extension.type) { +#define HANDLE_VARINT_TYPE(UPPERCASE, CPP_CAMELCASE) \ + case WireFormatLite::TYPE_##UPPERCASE: { \ + uint64 value; \ + ptr = VarintParse(ptr, &value); \ + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); \ + if (extension.is_repeated) { \ + Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, \ + extension.is_packed, value, extension.descriptor); \ + } else { \ + Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value, \ + extension.descriptor); \ + } \ + } break + + HANDLE_VARINT_TYPE(INT32, Int32); + HANDLE_VARINT_TYPE(INT64, Int64); + HANDLE_VARINT_TYPE(UINT32, UInt32); + HANDLE_VARINT_TYPE(UINT64, UInt64); + HANDLE_VARINT_TYPE(BOOL, Bool); +#undef HANDLE_VARINT_TYPE +#define HANDLE_SVARINT_TYPE(UPPERCASE, CPP_CAMELCASE, SIZE) \ + case WireFormatLite::TYPE_##UPPERCASE: { \ + uint64 val; \ + ptr = VarintParse(ptr, &val); \ + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); \ + auto value = WireFormatLite::ZigZagDecode##SIZE(val); \ + if (extension.is_repeated) { \ + Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, \ + extension.is_packed, value, extension.descriptor); \ + } else { \ + Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value, \ + extension.descriptor); \ + } \ + } break + + HANDLE_SVARINT_TYPE(SINT32, Int32, 32); + HANDLE_SVARINT_TYPE(SINT64, Int64, 64); +#undef HANDLE_SVARINT_TYPE +#define HANDLE_FIXED_TYPE(UPPERCASE, CPP_CAMELCASE, CPPTYPE) \ + case WireFormatLite::TYPE_##UPPERCASE: { \ + auto value = UnalignedLoad(ptr); \ + ptr += sizeof(CPPTYPE); \ + if (extension.is_repeated) { \ + Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, \ + extension.is_packed, value, extension.descriptor); \ + } else { \ + Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value, \ + extension.descriptor); \ + } \ + } break + + HANDLE_FIXED_TYPE(FIXED32, UInt32, uint32); + HANDLE_FIXED_TYPE(FIXED64, UInt64, uint64); + HANDLE_FIXED_TYPE(SFIXED32, Int32, int32); + HANDLE_FIXED_TYPE(SFIXED64, Int64, int64); + HANDLE_FIXED_TYPE(FLOAT, Float, float); + HANDLE_FIXED_TYPE(DOUBLE, Double, double); +#undef HANDLE_FIXED_TYPE + + case WireFormatLite::TYPE_ENUM: { + uint64 val; + ptr = VarintParse(ptr, &val); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + int value = val; + + if (!extension.enum_validity_check.func( + extension.enum_validity_check.arg, value)) { + WriteVarint(number, val, metadata->mutable_unknown_fields()); + } else if (extension.is_repeated) { + AddEnum(number, WireFormatLite::TYPE_ENUM, extension.is_packed, value, + extension.descriptor); + } else { + SetEnum(number, WireFormatLite::TYPE_ENUM, value, + extension.descriptor); + } + break; + } + + case WireFormatLite::TYPE_BYTES: + case WireFormatLite::TYPE_STRING: { + std::string* value = + extension.is_repeated + ? AddString(number, WireFormatLite::TYPE_STRING, + extension.descriptor) + : MutableString(number, WireFormatLite::TYPE_STRING, + extension.descriptor); + int size = ReadSize(&ptr); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + return ctx->ReadString(ptr, size, value); + } + + case WireFormatLite::TYPE_GROUP: { + MessageLite* value = + extension.is_repeated + ? AddMessage(number, WireFormatLite::TYPE_GROUP, + *extension.message_info.prototype, + extension.descriptor) + : MutableMessage(number, WireFormatLite::TYPE_GROUP, + *extension.message_info.prototype, + extension.descriptor); + uint32 tag = (number << 3) + WireFormatLite::WIRETYPE_START_GROUP; + return ctx->ParseGroup(value, ptr, tag); + } + + case WireFormatLite::TYPE_MESSAGE: { + MessageLite* value = + extension.is_repeated + ? AddMessage(number, WireFormatLite::TYPE_MESSAGE, + *extension.message_info.prototype, + extension.descriptor) + : MutableMessage(number, WireFormatLite::TYPE_MESSAGE, + *extension.message_info.prototype, + extension.descriptor); + return ctx->ParseMessage(value, ptr); + } + } + } + return ptr; +} + +template +const char* ExtensionSet::ParseMessageSetItemTmpl( + const char* ptr, const Msg* containing_type, + internal::InternalMetadata* metadata, internal::ParseContext* ctx) { + std::string payload; + uint32 type_id = 0; + while (!ctx->Done(&ptr)) { + uint32 tag = static_cast(*ptr++); + if (tag == WireFormatLite::kMessageSetTypeIdTag) { + uint64 tmp; + ptr = ParseBigVarint(ptr, &tmp); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + type_id = tmp; + if (!payload.empty()) { + ExtensionInfo extension; + bool was_packed_on_wire; + if (!FindExtension(2, type_id, containing_type, ctx, &extension, + &was_packed_on_wire)) { + WriteLengthDelimited(type_id, payload, + metadata->mutable_unknown_fields()); + } else { + MessageLite* value = + extension.is_repeated + ? AddMessage(type_id, WireFormatLite::TYPE_MESSAGE, + *extension.message_info.prototype, + extension.descriptor) + : MutableMessage(type_id, WireFormatLite::TYPE_MESSAGE, + *extension.message_info.prototype, + extension.descriptor); + + const char* p; + // We can't use regular parse from string as we have to track + // proper recursion depth and descriptor pools. + ParseContext tmp_ctx(ctx->depth(), false, &p, payload); + tmp_ctx.data().pool = ctx->data().pool; + tmp_ctx.data().factory = ctx->data().factory; + GOOGLE_PROTOBUF_PARSER_ASSERT(value->_InternalParse(p, &tmp_ctx) && + tmp_ctx.EndedAtLimit()); + } + type_id = 0; + } + } else if (tag == WireFormatLite::kMessageSetMessageTag) { + if (type_id != 0) { + ptr = ParseFieldMaybeLazily(static_cast(type_id) * 8 + 2, ptr, + containing_type, metadata, ctx); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr); + type_id = 0; + } else { + int32 size = ReadSize(&ptr); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + ptr = ctx->ReadString(ptr, size, &payload); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + } + } else { + ptr = ReadTag(ptr - 1, &tag); + if (tag == 0 || (tag & 7) == 4) { + ctx->SetLastTag(tag); + return ptr; + } + ptr = ParseField(tag, ptr, containing_type, metadata, ctx); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + } + } + return ptr; +} + +} // namespace internal +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__ diff --git a/third_party/protobuf-lite/google/protobuf/generated_enum_util.h b/third_party/protobuf-lite/google/protobuf/generated_enum_util.h index 96b03cc9..f1002e2d 100644 --- a/third_party/protobuf-lite/google/protobuf/generated_enum_util.h +++ b/third_party/protobuf-lite/google/protobuf/generated_enum_util.h @@ -33,14 +33,51 @@ #include +#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + namespace google { namespace protobuf { // This type trait can be used to cause templates to only match proto2 enum // types. -template struct is_proto_enum : ::std::false_type {}; +template +struct is_proto_enum : ::std::false_type {}; -} // namespace protobuf +namespace internal { + +// The table entry format for storing enum name-to-value mapping used with lite +// protos. This struct and the following related functions should only be used +// by protobuf generated code. +struct EnumEntry { + StringPiece name; + int value; +}; + +// Looks up a numeric enum value given the string name. +PROTOBUF_EXPORT bool LookUpEnumValue(const EnumEntry* enums, size_t size, + StringPiece name, int* value); +// Looks up an enum name given the numeric value. +PROTOBUF_EXPORT int LookUpEnumName(const EnumEntry* enums, + const int* sorted_indices, size_t size, + int value); + +// Initializes the list of enum names in std::string form. +PROTOBUF_EXPORT bool InitializeEnumStrings( + const EnumEntry* enums, const int* sorted_indices, size_t size, + internal::ExplicitlyConstructed* enum_strings); + +} // namespace internal +} // namespace protobuf } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_GENERATED_ENUM_UTIL_H__ diff --git a/third_party/protobuf-lite/google/protobuf/generated_message_reflection.h b/third_party/protobuf-lite/google/protobuf/generated_message_reflection.h new file mode 100644 index 00000000..e2eae772 --- /dev/null +++ b/third_party/protobuf-lite/google/protobuf/generated_message_reflection.h @@ -0,0 +1,317 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// This header is logically internal, but is made public because it is used +// from protocol-compiler-generated code, which may reside in other components. + +#ifndef GOOGLE_PROTOBUF_GENERATED_MESSAGE_REFLECTION_H__ +#define GOOGLE_PROTOBUF_GENERATED_MESSAGE_REFLECTION_H__ + +#include +#include +#include +#include +// TODO(jasonh): Remove this once the compiler change to directly include this +// is released to components. +#include +#include +#include +#include +#include + + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + +namespace google { +namespace protobuf { +class DescriptorPool; +class MapKey; +class MapValueRef; +class MessageLayoutInspector; +class Message; +struct Metadata; +} // namespace protobuf +} // namespace google + +namespace google { +namespace protobuf { +namespace internal { +class DefaultEmptyOneof; +// Defined in other files. +class ExtensionSet; // extension_set.h +class WeakFieldMap; // weak_field_map.h + +// This struct describes the internal layout of the message, hence this is +// used to act on the message reflectively. +// default_instance: The default instance of the message. This is only +// used to obtain pointers to default instances of embedded +// messages, which GetMessage() will return if the particular +// sub-message has not been initialized yet. (Thus, all +// embedded message fields *must* have non-null pointers +// in the default instance.) +// offsets: An array of ints giving the byte offsets. +// For each oneof or weak field, the offset is relative to the +// default_instance. These can be computed at compile time +// using the +// PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET() +// macro. For each none oneof field, the offset is related to +// the start of the message object. These can be computed at +// compile time using the +// PROTO2_GENERATED_MESSAGE_FIELD_OFFSET() macro. +// Besides offsets for all fields, this array also contains +// offsets for oneof unions. The offset of the i-th oneof union +// is offsets[descriptor->field_count() + i]. +// has_bit_indices: Mapping from field indexes to their index in the has +// bit array. +// has_bits_offset: Offset in the message of an array of uint32s of size +// descriptor->field_count()/32, rounded up. This is a +// bitfield where each bit indicates whether or not the +// corresponding field of the message has been initialized. +// The bit for field index i is obtained by the expression: +// has_bits[i / 32] & (1 << (i % 32)) +// unknown_fields_offset: Offset in the message of the UnknownFieldSet for +// the message. +// extensions_offset: Offset in the message of the ExtensionSet for the +// message, or -1 if the message type has no extension +// ranges. +// oneof_case_offset: Offset in the message of an array of uint32s of +// size descriptor->oneof_decl_count(). Each uint32 +// indicates what field is set for each oneof. +// object_size: The size of a message object of this type, as measured +// by sizeof(). +// arena_offset: If a message doesn't have a unknown_field_set that stores +// the arena, it must have a direct pointer to the arena. +// weak_field_map_offset: If the message proto has weak fields, this is the +// offset of _weak_field_map_ in the generated proto. Otherwise +// -1. +struct ReflectionSchema { + public: + // Size of a google::protobuf::Message object of this type. + uint32 GetObjectSize() const { return static_cast(object_size_); } + + bool InRealOneof(const FieldDescriptor* field) const { + return field->containing_oneof() && + !field->containing_oneof()->is_synthetic(); + } + + // Offset of a non-oneof field. Getting a field offset is slightly more + // efficient when we know statically that it is not a oneof field. + uint32 GetFieldOffsetNonOneof(const FieldDescriptor* field) const { + GOOGLE_DCHECK(!InRealOneof(field)); + return OffsetValue(offsets_[field->index()], field->type()); + } + + // Offset of any field. + uint32 GetFieldOffset(const FieldDescriptor* field) const { + if (InRealOneof(field)) { + size_t offset = + static_cast(field->containing_type()->field_count() + + field->containing_oneof()->index()); + return OffsetValue(offsets_[offset], field->type()); + } else { + return GetFieldOffsetNonOneof(field); + } + } + + bool IsFieldInlined(const FieldDescriptor* field) const { + if (InRealOneof(field)) { + size_t offset = + static_cast(field->containing_type()->field_count() + + field->containing_oneof()->index()); + return Inlined(offsets_[offset], field->type()); + } else { + return Inlined(offsets_[field->index()], field->type()); + } + } + + uint32 GetOneofCaseOffset(const OneofDescriptor* oneof_descriptor) const { + return static_cast(oneof_case_offset_) + + static_cast(static_cast(oneof_descriptor->index()) * + sizeof(uint32)); + } + + bool HasHasbits() const { return has_bits_offset_ != -1; } + + // Bit index within the bit array of hasbits. Bit order is low-to-high. + uint32 HasBitIndex(const FieldDescriptor* field) const { + if (has_bits_offset_ == -1) return static_cast(-1); + GOOGLE_DCHECK(HasHasbits()); + return has_bit_indices_[field->index()]; + } + + // Byte offset of the hasbits array. + uint32 HasBitsOffset() const { + GOOGLE_DCHECK(HasHasbits()); + return static_cast(has_bits_offset_); + } + + // The offset of the InternalMetadataWithArena member. + // For Lite this will actually be an InternalMetadataWithArenaLite. + // The schema doesn't contain enough information to distinguish between + // these two cases. + uint32 GetMetadataOffset() const { + return static_cast(metadata_offset_); + } + + // Whether this message has an ExtensionSet. + bool HasExtensionSet() const { return extensions_offset_ != -1; } + + // The offset of the ExtensionSet in this message. + uint32 GetExtensionSetOffset() const { + GOOGLE_DCHECK(HasExtensionSet()); + return static_cast(extensions_offset_); + } + + // The off set of WeakFieldMap when the message contains weak fields. + // The default is 0 for now. + int GetWeakFieldMapOffset() const { return weak_field_map_offset_; } + + bool IsDefaultInstance(const Message& message) const { + return &message == default_instance_; + } + + // Returns a pointer to the default value for this field. The size and type + // of the underlying data depends on the field's type. + const void* GetFieldDefault(const FieldDescriptor* field) const { + return reinterpret_cast(default_instance_) + + OffsetValue(offsets_[field->index()], field->type()); + } + + + bool HasWeakFields() const { return weak_field_map_offset_ > 0; } + + // These members are intended to be private, but we cannot actually make them + // private because this prevents us from using aggregate initialization of + // them, ie. + // + // ReflectionSchema schema = {a, b, c, d, e, ...}; + // private: + const Message* default_instance_; + const uint32* offsets_; + const uint32* has_bit_indices_; + int has_bits_offset_; + int metadata_offset_; + int extensions_offset_; + int oneof_case_offset_; + int object_size_; + int weak_field_map_offset_; + + // We tag offset values to provide additional data about fields (such as + // inlined). + static uint32 OffsetValue(uint32 v, FieldDescriptor::Type type) { + if (type == FieldDescriptor::TYPE_STRING || + type == FieldDescriptor::TYPE_BYTES) { + return v & ~1u; + } else { + return v; + } + } + + static bool Inlined(uint32 v, FieldDescriptor::Type type) { + if (type == FieldDescriptor::TYPE_STRING || + type == FieldDescriptor::TYPE_BYTES) { + return v & 1u; + } else { + // Non string/byte fields are not inlined. + return false; + } + } +}; + +// Structs that the code generator emits directly to describe a message. +// These should never used directly except to build a ReflectionSchema +// object. +// +// EXPERIMENTAL: these are changing rapidly, and may completely disappear +// or merge with ReflectionSchema. +struct MigrationSchema { + int32 offsets_index; + int32 has_bit_indices_index; + int object_size; +}; + +struct SCCInfoBase; + +struct PROTOBUF_EXPORT DescriptorTable { + mutable bool is_initialized; + bool is_eager; + const char* descriptor; + const char* filename; + int size; // of serialized descriptor + once_flag* once; + SCCInfoBase* const* init_default_instances; + const DescriptorTable* const* deps; + int num_sccs; + int num_deps; + const MigrationSchema* schemas; + const Message* const* default_instances; + const uint32* offsets; + // update the following descriptor arrays. + Metadata* file_level_metadata; + int num_messages; + const EnumDescriptor** file_level_enum_descriptors; + const ServiceDescriptor** file_level_service_descriptors; +}; + +// AssignDescriptors() pulls the compiled FileDescriptor from the DescriptorPool +// and uses it to populate all of the global variables which store pointers to +// the descriptor objects. It also constructs the reflection objects. It is +// called the first time anyone calls descriptor() or GetReflection() on one of +// the types defined in the file. AssignDescriptors() is thread-safe. +void PROTOBUF_EXPORT AssignDescriptors(const DescriptorTable* table, + bool eager = false); + +// AddDescriptors() is a file-level procedure which adds the encoded +// FileDescriptorProto for this .proto file to the global DescriptorPool for +// generated files (DescriptorPool::generated_pool()). It ordinarily runs at +// static initialization time, but is not used at all in LITE_RUNTIME mode. +// AddDescriptors() is *not* thread-safe. +void PROTOBUF_EXPORT AddDescriptors(const DescriptorTable* table); + +// These cannot be in lite so we put them in the reflection. +PROTOBUF_EXPORT void UnknownFieldSetSerializer(const uint8* base, uint32 offset, + uint32 tag, uint32 has_offset, + io::CodedOutputStream* output); + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include + +#endif // GOOGLE_PROTOBUF_GENERATED_MESSAGE_REFLECTION_H__ diff --git a/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h b/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h index 10ca3aaa..1394ab14 100644 --- a/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h +++ b/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h @@ -36,7 +36,6 @@ #include #include #include -#include // We require C++11 and Clang to use constexpr for variables, as GCC 4.8 // requires constexpr to be consistent between declarations of variables @@ -48,6 +47,12 @@ #define PROTOBUF_CONSTEXPR_VAR #endif // !_clang +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + +#include + namespace google { namespace protobuf { namespace internal { @@ -64,6 +69,10 @@ static constexpr const unsigned char kNotPackedMask = 0x10; static constexpr const unsigned char kInvalidMask = 0x20; enum ProcessingTypes { + TYPE_STRING_CORD = 19, + TYPE_STRING_STRING_PIECE = 20, + TYPE_BYTES_CORD = 21, + TYPE_BYTES_STRING_PIECE = 22, TYPE_STRING_INLINED = 23, TYPE_BYTES_INLINED = 24, TYPE_MAP = 25, @@ -71,6 +80,40 @@ enum ProcessingTypes { static_assert(TYPE_MAP < kRepeatedMask, "Invalid enum"); +struct PROTOBUF_EXPORT FieldMetadata { + uint32 offset; // offset of this field in the struct + uint32 tag; // field * 8 + wire_type + // byte offset * 8 + bit_offset; + // if the high bit is set then this is the byte offset of the oneof_case + // for this field. + uint32 has_offset; + uint32 type; // the type of this field. + const void* ptr; // auxiliary data + + // From the serializer point of view each fundamental type can occur in + // 4 different ways. For simplicity we treat all combinations as a cartesion + // product although not all combinations are allowed. + enum FieldTypeClass { + kPresence, + kNoPresence, + kRepeated, + kPacked, + kOneOf, + kNumTypeClasses // must be last enum + }; + // C++ protobuf has 20 fundamental types, were we added Cord and StringPiece + // and also distinquish the same types if they have different wire format. + enum { + kCordType = 19, + kStringPieceType = 20, + kInlinedType = 21, + kNumTypes = 21, + kSpecial = kNumTypes * kNumTypeClasses, + }; + + static int CalculateType(int fundamental_type, FieldTypeClass type_class); +}; + // TODO(ckennelly): Add a static assertion to ensure that these masks do not // conflict with wiretypes. @@ -164,37 +207,133 @@ static_assert(sizeof(ParseTableField) <= 16, "ParseTableField is too large"); // The tables must be composed of POD components to ensure link-time // initialization. static_assert(std::is_pod::value, ""); +static_assert(std::is_pod::value, ""); static_assert(std::is_pod::value, ""); static_assert(std::is_pod::value, ""); static_assert(std::is_pod::value, ""); static_assert(std::is_pod::value, ""); -#ifndef __NVCC__ // This assertion currently fails under NVCC. -static_assert(std::is_pod::value, ""); -#endif - // TODO(ckennelly): Consolidate these implementations into a single one, using // dynamic dispatch to the appropriate unknown field handler. bool MergePartialFromCodedStream(MessageLite* msg, const ParseTable& table, io::CodedInputStream* input); bool MergePartialFromCodedStreamLite(MessageLite* msg, const ParseTable& table, - io::CodedInputStream* input); + io::CodedInputStream* input); template bool ParseMap(io::CodedInputStream* input, void* map_field) { typedef typename MapEntryToMapField::MapFieldType MapFieldType; - typedef google::protobuf::Map + typedef Map MapType; typedef typename Entry::template Parser ParserType; ParserType parser(static_cast(map_field)); - return ::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(input, - &parser); + return WireFormatLite::ReadMessageNoVirtual(input, &parser); +} + +struct SerializationTable { + int num_fields; + const FieldMetadata* field_table; +}; + +PROTOBUF_EXPORT void SerializeInternal(const uint8* base, + const FieldMetadata* table, + int32 num_fields, + io::CodedOutputStream* output); + +inline void TableSerialize(const MessageLite& msg, + const SerializationTable* table, + io::CodedOutputStream* output) { + const FieldMetadata* field_table = table->field_table; + int num_fields = table->num_fields - 1; + const uint8* base = reinterpret_cast(&msg); + // TODO(gerbens) This skips the first test if we could use the fast + // array serialization path, we should make this + // int cached_size = + // *reinterpret_cast(base + field_table->offset); + // SerializeWithCachedSize(msg, field_table + 1, num_fields, cached_size, ...) + // But we keep conformance with the old way for now. + SerializeInternal(base, field_table + 1, num_fields, output); +} + +uint8* SerializeInternalToArray(const uint8* base, const FieldMetadata* table, + int32 num_fields, bool is_deterministic, + uint8* buffer); + +inline uint8* TableSerializeToArray(const MessageLite& msg, + const SerializationTable* table, + bool is_deterministic, uint8* buffer) { + const uint8* base = reinterpret_cast(&msg); + const FieldMetadata* field_table = table->field_table + 1; + int num_fields = table->num_fields - 1; + return SerializeInternalToArray(base, field_table, num_fields, + is_deterministic, buffer); +} + +template +struct CompareHelper { + bool operator()(const T& a, const T& b) const { return a < b; } +}; + +template <> +struct CompareHelper { + bool operator()(const ArenaStringPtr& a, const ArenaStringPtr& b) const { + return a.Get() < b.Get(); + } +}; + +struct CompareMapKey { + template + bool operator()(const MapEntryHelper& a, + const MapEntryHelper& b) const { + return Compare(a.key_, b.key_); + } + template + bool Compare(const T& a, const T& b) const { + return CompareHelper()(a, b); + } +}; + +template +void MapFieldSerializer(const uint8* base, uint32 offset, uint32 tag, + uint32 has_offset, io::CodedOutputStream* output) { + typedef MapEntryHelper Entry; + typedef typename MapFieldType::MapType::const_iterator Iter; + + const MapFieldType& map_field = + *reinterpret_cast(base + offset); + const SerializationTable* t = + table + + has_offset; // has_offset is overloaded for maps to mean table offset + if (!output->IsSerializationDeterministic()) { + for (Iter it = map_field.GetMap().begin(); it != map_field.GetMap().end(); + ++it) { + Entry map_entry(*it); + output->WriteVarint32(tag); + output->WriteVarint32(map_entry._cached_size_); + SerializeInternal(reinterpret_cast(&map_entry), + t->field_table, t->num_fields, output); + } + } else { + std::vector v; + for (Iter it = map_field.GetMap().begin(); it != map_field.GetMap().end(); + ++it) { + v.push_back(Entry(*it)); + } + std::sort(v.begin(), v.end(), CompareMapKey()); + for (int i = 0; i < v.size(); i++) { + output->WriteVarint32(tag); + output->WriteVarint32(v[i]._cached_size_); + SerializeInternal(reinterpret_cast(&v[i]), t->field_table, + t->num_fields, output); + } + } } } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_GENERATED_MESSAGE_TABLE_DRIVEN_H__ diff --git a/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h b/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h index 0d90fe33..ae13b363 100644 --- a/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h +++ b/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h @@ -37,13 +37,13 @@ #include #include #include -#include #include #include -#include #include +#include + namespace google { namespace protobuf { namespace internal { @@ -81,15 +81,6 @@ inline const Type* Raw(const MessageLite* msg, int64 offset) { offset); } -template -inline Arena* GetArena(MessageLite* msg, int64 arena_offset) { - if (GOOGLE_PREDICT_FALSE(arena_offset == -1)) { - return NULL; - } - - return Raw(msg, arena_offset)->arena(); -} - inline ExtensionSet* GetExtensionSet(MessageLite* msg, int64 extension_offset) { if (extension_offset == -1) { return NULL; @@ -101,18 +92,17 @@ inline ExtensionSet* GetExtensionSet(MessageLite* msg, int64 extension_offset) { template inline Type* AddField(MessageLite* msg, int64 offset) { static_assert(std::is_pod::value || - std::is_same::value, + std::is_same::value, "Do not assign"); - google::protobuf::RepeatedField* repeated = - Raw >(msg, offset); + RepeatedField* repeated = Raw>(msg, offset); return repeated->Add(); } template <> -inline string* AddField(MessageLite* msg, int64 offset) { - google::protobuf::RepeatedPtrField* repeated = - Raw >(msg, offset); +inline std::string* AddField(MessageLite* msg, int64 offset) { + RepeatedPtrField* repeated = + Raw>(msg, offset); return repeated->Add(); } @@ -157,7 +147,7 @@ inline void SetOneofField(MessageLite* msg, uint32* oneof_case, // Clears a oneof field. The field argument should correspond to the particular // field that is currently set in the oneof. inline void ClearOneofField(const ParseTableField& field, Arena* arena, - MessageLite* msg) { + MessageLite* msg) { switch (field.processing_type & kTypeMask) { case WireFormatLite::TYPE_MESSAGE: if (arena == NULL) { @@ -168,7 +158,7 @@ inline void ClearOneofField(const ParseTableField& field, Arena* arena, case WireFormatLite::TYPE_STRING: case WireFormatLite::TYPE_BYTES: Raw(msg, field.offset) - ->Destroy(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), arena); + ->Destroy(&GetEmptyStringAlreadyInited(), arena); break; case TYPE_STRING_INLINED: @@ -206,11 +196,11 @@ inline void ResetOneofField(const ParseTable& table, int field_number, switch (field_type) { case ProcessingType_STRING: Raw(msg, offset) - ->UnsafeSetDefault(static_cast(default_ptr)); + ->UnsafeSetDefault(static_cast(default_ptr)); break; case ProcessingType_INLINED: new (Raw(msg, offset)) - InlinedStringField(*static_cast(default_ptr)); + InlinedStringField(*static_cast(default_ptr)); break; case ProcessingType_MESSAGE: MessageLite** submessage = Raw(msg, offset); @@ -221,25 +211,28 @@ inline void ResetOneofField(const ParseTable& table, int field_number, } } -template +template static inline bool HandleString(io::CodedInputStream* input, MessageLite* msg, Arena* arena, uint32* has_bits, uint32 has_bit_index, int64 offset, const void* default_ptr, const char* field_name) { + StringPiece utf8_string_data; #ifdef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED - const char* sdata; - size_t size; -#endif + constexpr bool kValidateUtf8 = is_string_type; +#else + constexpr bool kValidateUtf8 = false; +#endif // GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED switch (ctype) { case StringType_INLINED: { - InlinedStringField* s; + InlinedStringField* s = nullptr; switch (cardinality) { case Cardinality_SINGULAR: // TODO(ckennelly): Is this optimal? - s = MutableField( - msg, has_bits, has_bit_index, offset); + s = MutableField(msg, has_bits, has_bit_index, + offset); break; case Cardinality_REPEATED: s = AddField(msg, offset); @@ -249,67 +242,69 @@ static inline bool HandleString(io::CodedInputStream* input, MessageLite* msg, break; } GOOGLE_DCHECK(s != nullptr); - ::std::string* value = s->MutableNoArena(NULL); - - if (GOOGLE_PREDICT_FALSE(!WireFormatLite::ReadString(input, value))) { + std::string* value = s->MutableNoArena(NULL); + if (PROTOBUF_PREDICT_FALSE(!WireFormatLite::ReadString(input, value))) { return false; } - -#ifdef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED - sdata = value->data(); - size = value->size(); -#endif + utf8_string_data = *value; break; } case StringType_STRING: { - string* value; switch (cardinality) { - case Cardinality_SINGULAR: - // TODO(ckennelly): Is this optimal? - value = - MutableField(msg, has_bits, has_bit_index, offset) - ->Mutable(static_cast(default_ptr), arena); - break; - case Cardinality_REPEATED: - value = AddField(msg, offset); - break; - case Cardinality_ONEOF: - value = Raw(msg, offset) - ->Mutable(static_cast(default_ptr), arena); - break; - } - GOOGLE_DCHECK(value != nullptr); - - if (GOOGLE_PREDICT_FALSE(!WireFormatLite::ReadString(input, value))) { - return false; + case Cardinality_SINGULAR: { + ArenaStringPtr* field = MutableField( + msg, has_bits, has_bit_index, offset); + std::string* value = field->Mutable( + static_cast(default_ptr), arena); + if (PROTOBUF_PREDICT_FALSE( + !WireFormatLite::ReadString(input, value))) { + return false; + } + utf8_string_data = field->Get(); + } break; + case Cardinality_REPEATED: { + std::string* value = AddField(msg, offset); + if (PROTOBUF_PREDICT_FALSE( + !WireFormatLite::ReadString(input, value))) { + return false; + } + utf8_string_data = *value; + } break; + case Cardinality_ONEOF: { + ArenaStringPtr* field = Raw(msg, offset); + std::string* value = field->Mutable( + static_cast(default_ptr), arena); + if (PROTOBUF_PREDICT_FALSE( + !WireFormatLite::ReadString(input, value))) { + return false; + } + utf8_string_data = field->Get(); + } break; + default: + PROTOBUF_ASSUME(false); } - -#ifdef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED - sdata = value->data(); - size = value->size(); -#endif break; } + default: + PROTOBUF_ASSUME(false); } -#ifdef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED - if (validate) { - WireFormatLite::VerifyUtf8String(sdata, size, WireFormatLite::PARSE, - field_name); + if (kValidateUtf8) { + // TODO(b/118759213): fail if proto3 + WireFormatLite::VerifyUtf8String(utf8_string_data.data(), + utf8_string_data.length(), + WireFormatLite::PARSE, field_name); } -#endif - return true; } -template +template inline bool HandleEnum(const ParseTable& table, io::CodedInputStream* input, MessageLite* msg, uint32* presence, uint32 presence_index, int64 offset, uint32 tag, int field_number) { int value; - if (GOOGLE_PREDICT_FALSE( + if (PROTOBUF_PREDICT_FALSE( (!WireFormatLite::ReadPrimitive( input, &value)))) { return false; @@ -317,7 +312,7 @@ inline bool HandleEnum(const ParseTable& table, io::CodedInputStream* input, AuxillaryParseTableField::EnumValidator validator = table.aux[field_number].enums.validator; - if (validator(value)) { + if (validator == nullptr || validator(value)) { switch (cardinality) { case Cardinality_SINGULAR: SetField(msg, presence, presence_index, offset, value); @@ -326,12 +321,13 @@ inline bool HandleEnum(const ParseTable& table, io::CodedInputStream* input, AddField(msg, offset, value); break; case Cardinality_ONEOF: - ClearOneofField(table.fields[presence[presence_index]], - GetArena(msg, table.arena_offset), + ClearOneofField(table.fields[presence[presence_index]], msg->GetArena(), msg); SetOneofField(msg, presence, presence_index, offset, field_number, value); break; + default: + PROTOBUF_ASSUME(false); } } else { UnknownFieldHandler::Varint(msg, table, tag, value); @@ -370,9 +366,10 @@ class MergePartialFromCodedStreamHelper { } }; -template -bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, - io::CodedInputStream* input) { +template +bool MergePartialFromCodedStreamInlined(MessageLite* msg, + const ParseTable& table, + io::CodedInputStream* input) { // We require that has_bits are present, as to avoid having to check for them // for every field. // @@ -382,20 +379,19 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, GOOGLE_DCHECK(has_bits != NULL); while (true) { - uint32 tag = input->ReadTag(); - + uint32 tag = input->ReadTagWithCutoffNoLastTag(kMaxTag).first; const WireFormatLite::WireType wire_type = WireFormatLite::GetTagWireType(tag); const int field_number = WireFormatLite::GetTagFieldNumber(tag); - if (field_number > table.max_field_number) { + if (PROTOBUF_PREDICT_FALSE(field_number > table.max_field_number)) { // check for possible extensions if (UnknownFieldHandler::ParseExtension(msg, table, input, tag)) { // successfully parsed continue; } - if (GOOGLE_PREDICT_FALSE( + if (PROTOBUF_PREDICT_FALSE( !UnknownFieldHandler::Skip(msg, table, input, tag))) { return false; } @@ -416,14 +412,11 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, const unsigned char processing_type = data->processing_type; if (data->normal_wiretype == static_cast(wire_type)) { - // TODO(ckennelly): Use a computed goto on GCC/LLVM or otherwise eliminate - // the bounds check on processing_type. - switch (processing_type) { #define HANDLE_TYPE(TYPE, CPPTYPE) \ case (WireFormatLite::TYPE_##TYPE): { \ CPPTYPE value; \ - if (GOOGLE_PREDICT_FALSE( \ + if (PROTOBUF_PREDICT_FALSE( \ (!WireFormatLite::ReadPrimitive< \ CPPTYPE, WireFormatLite::TYPE_##TYPE>(input, &value)))) { \ return false; \ @@ -432,10 +425,9 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, break; \ } \ case (WireFormatLite::TYPE_##TYPE) | kRepeatedMask: { \ - google::protobuf::RepeatedField* values = \ - Raw >(msg, offset); \ - if (GOOGLE_PREDICT_FALSE((!WireFormatLite::ReadRepeatedPrimitive< \ - CPPTYPE, WireFormatLite::TYPE_##TYPE>( \ + RepeatedField* values = Raw>(msg, offset); \ + if (PROTOBUF_PREDICT_FALSE((!WireFormatLite::ReadRepeatedPrimitive< \ + CPPTYPE, WireFormatLite::TYPE_##TYPE>( \ data->tag_size, tag, input, values)))) { \ return false; \ } \ @@ -444,13 +436,13 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, case (WireFormatLite::TYPE_##TYPE) | kOneofMask: { \ uint32* oneof_case = Raw(msg, table.oneof_case_offset); \ CPPTYPE value; \ - if (GOOGLE_PREDICT_FALSE( \ + if (PROTOBUF_PREDICT_FALSE( \ (!WireFormatLite::ReadPrimitive< \ CPPTYPE, WireFormatLite::TYPE_##TYPE>(input, &value)))) { \ return false; \ } \ - ClearOneofField(table.fields[oneof_case[presence_index]], \ - GetArena(msg, table.arena_offset), msg); \ + ClearOneofField(table.fields[oneof_case[presence_index]], msg->GetArena(), \ + msg); \ SetOneofField(msg, oneof_case, presence_index, offset, field_number, \ value); \ break; \ @@ -476,16 +468,16 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, case WireFormatLite::TYPE_BYTES: #ifndef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case WireFormatLite::TYPE_STRING: -#endif +#endif // GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); const void* default_ptr = table.aux[field_number].strings.default_ptr; - if (GOOGLE_PREDICT_FALSE(( - !HandleString( + if (PROTOBUF_PREDICT_FALSE( + (!HandleString( input, msg, arena, has_bits, presence_index, offset, - default_ptr, NULL)))) { + default_ptr, nullptr)))) { return false; } break; @@ -493,16 +485,16 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, case TYPE_BYTES_INLINED: #ifndef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case TYPE_STRING_INLINED: -#endif +#endif // !GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); const void* default_ptr = table.aux[field_number].strings.default_ptr; - if (GOOGLE_PREDICT_FALSE((!HandleString( - input, msg, arena, has_bits, presence_index, offset, - default_ptr, NULL)))) { + if (PROTOBUF_PREDICT_FALSE( + (!HandleString( + input, msg, arena, has_bits, presence_index, offset, + default_ptr, nullptr)))) { return false; } break; @@ -510,10 +502,9 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, case WireFormatLite::TYPE_BYTES | kOneofMask: #ifndef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case WireFormatLite::TYPE_STRING | kOneofMask: -#endif +#endif // !GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); uint32* oneof_case = Raw(msg, table.oneof_case_offset); const void* default_ptr = table.aux[field_number].strings.default_ptr; @@ -521,10 +512,11 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, table, field_number, arena, msg, oneof_case + presence_index, offset, default_ptr); - if (GOOGLE_PREDICT_FALSE( - (!HandleString( - input, msg, arena, has_bits, presence_index, offset, - default_ptr, NULL)))) { + if (PROTOBUF_PREDICT_FALSE( + (!HandleString(input, msg, arena, has_bits, + presence_index, offset, + default_ptr, nullptr)))) { return false; } break; @@ -534,30 +526,29 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, #ifndef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case (WireFormatLite::TYPE_STRING) | kRepeatedMask: case TYPE_STRING_INLINED | kRepeatedMask: -#endif +#endif // !GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED { - Arena* const arena = - GetArena(msg, table.arena_offset); - const void* default_ptr = - table.aux[field_number].strings.default_ptr; + Arena* const arena = msg->GetArena(); + const void* default_ptr = table.aux[field_number].strings.default_ptr; - if (GOOGLE_PREDICT_FALSE(( - !HandleString( + if (PROTOBUF_PREDICT_FALSE( + (!HandleString( input, msg, arena, has_bits, presence_index, offset, - default_ptr, NULL)))) { + default_ptr, nullptr)))) { return false; } break; } #ifdef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case (WireFormatLite::TYPE_STRING): { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); const void* default_ptr = table.aux[field_number].strings.default_ptr; const char* field_name = table.aux[field_number].strings.field_name; - if (GOOGLE_PREDICT_FALSE( - (!HandleString( + if (PROTOBUF_PREDICT_FALSE( + (!HandleString( input, msg, arena, has_bits, presence_index, offset, default_ptr, field_name)))) { return false; @@ -566,13 +557,13 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, } case TYPE_STRING_INLINED | kRepeatedMask: case (WireFormatLite::TYPE_STRING) | kRepeatedMask: { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); const void* default_ptr = table.aux[field_number].strings.default_ptr; const char* field_name = table.aux[field_number].strings.field_name; - if (GOOGLE_PREDICT_FALSE( - (!HandleString( + if (PROTOBUF_PREDICT_FALSE( + (!HandleString( input, msg, arena, has_bits, presence_index, offset, default_ptr, field_name)))) { return false; @@ -580,8 +571,7 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, break; } case (WireFormatLite::TYPE_STRING) | kOneofMask: { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); uint32* oneof_case = Raw(msg, table.oneof_case_offset); const void* default_ptr = table.aux[field_number].strings.default_ptr; const char* field_name = table.aux[field_number].strings.field_name; @@ -590,19 +580,19 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, table, field_number, arena, msg, oneof_case + presence_index, offset, default_ptr); - if (GOOGLE_PREDICT_FALSE( - (!HandleString( + if (PROTOBUF_PREDICT_FALSE( + (!HandleString( input, msg, arena, has_bits, presence_index, offset, default_ptr, field_name)))) { return false; } break; } -#endif +#endif // GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case WireFormatLite::TYPE_ENUM: { - if (GOOGLE_PREDICT_FALSE( - (!HandleEnum( + if (PROTOBUF_PREDICT_FALSE( + (!HandleEnum( table, input, msg, has_bits, presence_index, offset, tag, field_number)))) { return false; @@ -610,9 +600,8 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, break; } case WireFormatLite::TYPE_ENUM | kRepeatedMask: { - if (GOOGLE_PREDICT_FALSE( - (!HandleEnum( + if (PROTOBUF_PREDICT_FALSE( + (!HandleEnum( table, input, msg, has_bits, presence_index, offset, tag, field_number)))) { return false; @@ -621,11 +610,10 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, } case WireFormatLite::TYPE_ENUM | kOneofMask: { uint32* oneof_case = Raw(msg, table.oneof_case_offset); - if (GOOGLE_PREDICT_FALSE( - (!HandleEnum(table, input, msg, oneof_case, - presence_index, offset, tag, - field_number)))) { + if (PROTOBUF_PREDICT_FALSE( + (!HandleEnum( + table, input, msg, oneof_case, presence_index, offset, + tag, field_number)))) { return false; } break; @@ -636,15 +624,14 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, MessageLite* submsg = *submsg_holder; if (submsg == NULL) { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); const MessageLite* prototype = table.aux[field_number].messages.default_message(); submsg = prototype->New(arena); *submsg_holder = submsg; } - if (GOOGLE_PREDICT_FALSE( + if (PROTOBUF_PREDICT_FALSE( !WireFormatLite::ReadGroup(field_number, input, submsg))) { return false; } @@ -660,7 +647,7 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, MessageLite* submsg = MergePartialFromCodedStreamHelper::Add(field, prototype); - if (GOOGLE_PREDICT_FALSE( + if (PROTOBUF_PREDICT_FALSE( !WireFormatLite::ReadGroup(field_number, input, submsg))) { return false; } @@ -673,19 +660,18 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, MessageLite* submsg = *submsg_holder; if (submsg == NULL) { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); const MessageLite* prototype = table.aux[field_number].messages.default_message(); if (prototype == NULL) { - prototype = - ::google::protobuf::internal::ImplicitWeakMessage::default_instance(); + prototype = ImplicitWeakMessage::default_instance(); } submsg = prototype->New(arena); *submsg_holder = submsg; } - if (GOOGLE_PREDICT_FALSE(!WireFormatLite::ReadMessage(input, submsg))) { + if (PROTOBUF_PREDICT_FALSE( + !WireFormatLite::ReadMessage(input, submsg))) { return false; } @@ -698,22 +684,21 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, const MessageLite* prototype = table.aux[field_number].messages.default_message(); if (prototype == NULL) { - prototype = - ::google::protobuf::internal::ImplicitWeakMessage::default_instance(); + prototype = ImplicitWeakMessage::default_instance(); } MessageLite* submsg = MergePartialFromCodedStreamHelper::Add(field, prototype); - if (GOOGLE_PREDICT_FALSE(!WireFormatLite::ReadMessage(input, submsg))) { + if (PROTOBUF_PREDICT_FALSE( + !WireFormatLite::ReadMessage(input, submsg))) { return false; } break; } case WireFormatLite::TYPE_MESSAGE | kOneofMask: { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); uint32* oneof_case = Raw(msg, table.oneof_case_offset); MessageLite** submsg_holder = Raw(msg, offset); ResetOneofField( @@ -721,7 +706,8 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, offset, NULL); MessageLite* submsg = *submsg_holder; - if (GOOGLE_PREDICT_FALSE(!WireFormatLite::ReadMessage(input, submsg))) { + if (PROTOBUF_PREDICT_FALSE( + !WireFormatLite::ReadMessage(input, submsg))) { return false; } @@ -729,13 +715,13 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, } #ifdef GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case TYPE_STRING_INLINED: { - Arena* const arena = - GetArena(msg, table.arena_offset); + Arena* const arena = msg->GetArena(); const void* default_ptr = table.aux[field_number].strings.default_ptr; const char* field_name = table.aux[field_number].strings.field_name; - if (GOOGLE_PREDICT_FALSE(( - !HandleString( + if (PROTOBUF_PREDICT_FALSE( + (!HandleString( input, msg, arena, has_bits, presence_index, offset, default_ptr, field_name)))) { return false; @@ -744,7 +730,7 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, } #endif // GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED case TYPE_MAP: { - if (GOOGLE_PREDICT_FALSE(!(*table.aux[field_number].maps.parse_map)( + if (PROTOBUF_PREDICT_FALSE(!(*table.aux[field_number].maps.parse_map)( input, Raw(msg, offset)))) { return false; } @@ -752,10 +738,11 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, } case 0: { // Done. + input->SetLastTag(tag); return true; } default: - break; + PROTOBUF_ASSUME(false); } } else if (data->packed_wiretype == static_cast(wire_type)) { // Non-packable fields have their packed_wiretype masked with @@ -767,21 +754,18 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, GOOGLE_DCHECK_NE(TYPE_BYTES_INLINED | kRepeatedMask, processing_type); GOOGLE_DCHECK_NE(TYPE_STRING_INLINED | kRepeatedMask, processing_type); - // TODO(ckennelly): Use a computed goto on GCC/LLVM. - // // Mask out kRepeatedMask bit, allowing the jump table to be smaller. - switch (static_cast( - processing_type ^ kRepeatedMask)) { -#define HANDLE_PACKED_TYPE(TYPE, CPPTYPE, CPPTYPE_METHOD) \ - case WireFormatLite::TYPE_##TYPE: { \ - google::protobuf::RepeatedField* values = \ - Raw >(msg, offset); \ - if (GOOGLE_PREDICT_FALSE( \ - (!WireFormatLite::ReadPackedPrimitive< \ - CPPTYPE, WireFormatLite::TYPE_##TYPE>(input, values)))) { \ - return false; \ - } \ - break; \ + switch (static_cast(processing_type ^ + kRepeatedMask)) { +#define HANDLE_PACKED_TYPE(TYPE, CPPTYPE, CPPTYPE_METHOD) \ + case WireFormatLite::TYPE_##TYPE: { \ + RepeatedField* values = Raw>(msg, offset); \ + if (PROTOBUF_PREDICT_FALSE( \ + (!WireFormatLite::ReadPackedPrimitive< \ + CPPTYPE, WireFormatLite::TYPE_##TYPE>(input, values)))) { \ + return false; \ + } \ + break; \ } HANDLE_PACKED_TYPE(INT32, int32, Int32) @@ -803,29 +787,28 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, #undef HANDLE_PACKED_TYPE case WireFormatLite::TYPE_ENUM: { // To avoid unnecessarily calling MutableUnknownFields (which mutates - // InternalMetadataWithArena) when all inputs in the repeated series + // InternalMetadata) when all inputs in the repeated series // are valid, we implement our own parser rather than call // WireFormat::ReadPackedEnumPreserveUnknowns. uint32 length; - if (GOOGLE_PREDICT_FALSE(!input->ReadVarint32(&length))) { + if (PROTOBUF_PREDICT_FALSE(!input->ReadVarint32(&length))) { return false; } AuxillaryParseTableField::EnumValidator validator = table.aux[field_number].enums.validator; - google::protobuf::RepeatedField* values = - Raw >(msg, offset); + RepeatedField* values = Raw>(msg, offset); io::CodedInputStream::Limit limit = input->PushLimit(length); while (input->BytesUntilLimit() > 0) { int value; - if (GOOGLE_PREDICT_FALSE( - (!google::protobuf::internal::WireFormatLite::ReadPrimitive< + if (PROTOBUF_PREDICT_FALSE( + (!WireFormatLite::ReadPrimitive< int, WireFormatLite::TYPE_ENUM>(input, &value)))) { return false; } - if (validator(value)) { + if (validator == nullptr || validator(value)) { values->Add(value); } else { // TODO(ckennelly): Consider caching here. @@ -843,11 +826,12 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, GOOGLE_DCHECK(false); return false; default: - break; + PROTOBUF_ASSUME(false); } } else { if (wire_type == WireFormatLite::WIRETYPE_END_GROUP) { // Must be the end of the message. + input->SetLastTag(tag); return true; } @@ -858,7 +842,7 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, } // process unknown field. - if (GOOGLE_PREDICT_FALSE( + if (PROTOBUF_PREDICT_FALSE( !UnknownFieldHandler::Skip(msg, table, input, tag))) { return false; } @@ -866,8 +850,28 @@ bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, } } +template +bool MergePartialFromCodedStreamImpl(MessageLite* msg, const ParseTable& table, + io::CodedInputStream* input) { + // The main beneficial cutoff values are 1 and 2 byte tags. + // Instantiate calls with the appropriate upper tag range + if (table.max_field_number <= (0x7F >> 3)) { + return MergePartialFromCodedStreamInlined( + msg, table, input); + } else if (table.max_field_number <= (0x3FFF >> 3)) { + return MergePartialFromCodedStreamInlined( + msg, table, input); + } else { + return MergePartialFromCodedStreamInlined< + UnknownFieldHandler, std::numeric_limits::max()>(msg, table, + input); + } +} + } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_GENERATED_MESSAGE_TABLE_DRIVEN_LITE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/generated_message_util.h b/third_party/protobuf-lite/google/protobuf/generated_message_util.h index 04f68a6e..7cae4e12 100644 --- a/third_party/protobuf-lite/google/protobuf/generated_message_util.h +++ b/third_party/protobuf-lite/google/protobuf/generated_message_util.h @@ -39,91 +39,69 @@ #define GOOGLE_PROTOBUF_GENERATED_MESSAGE_UTIL_H__ #include + #include #include #include #include -#include #include -#include // Add direct dep on port for pb.cc +#include #include #include -#include #include +#include // Add direct dep on port for pb.cc +#include +#include #include +#include +#include -namespace google { +#include +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + +namespace google { namespace protobuf { class Arena; +class Message; -namespace io { class CodedInputStream; } +namespace io { +class CodedInputStream; +} namespace internal { +template +inline To DownCast(From* f) { + return PROTOBUF_NAMESPACE_ID::internal::down_cast(f); +} +template +inline To DownCast(From& f) { + return PROTOBUF_NAMESPACE_ID::internal::down_cast(f); +} -// Annotation for the compiler to emit a deprecation message if a field marked -// with option 'deprecated=true' is used in the code, or for other things in -// generated code which are deprecated. -// -// For internal use in the pb.cc files, deprecation warnings are suppressed -// there. -#undef DEPRECATED_PROTOBUF_FIELD -#define PROTOBUF_DEPRECATED - -#define GOOGLE_PROTOBUF_DEPRECATED_ATTR - - -// Returns the offset of the given field within the given aggregate type. -// This is equivalent to the ANSI C offsetof() macro. However, according -// to the C++ standard, offsetof() only works on POD types, and GCC -// enforces this requirement with a warning. In practice, this rule is -// unnecessarily strict; there is probably no compiler or platform on -// which the offsets of the direct fields of a class are non-constant. -// Fields inherited from superclasses *can* have non-constant offsets, -// but that's not what this macro will be used for. -#if defined(__clang__) -// For Clang we use __builtin_offsetof() and suppress the warning, -// to avoid Control Flow Integrity and UBSan vptr sanitizers from -// crashing while trying to validate the invalid reinterpet_casts. -#define GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(TYPE, FIELD) \ - _Pragma("clang diagnostic push") \ - _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \ - __builtin_offsetof(TYPE, FIELD) \ - _Pragma("clang diagnostic pop") -#else -// Note that we calculate relative to the pointer value 16 here since if we -// just use zero, GCC complains about dereferencing a NULL pointer. We -// choose 16 rather than some other number just in case the compiler would -// be confused by an unaligned pointer. -#define GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(TYPE, FIELD) \ - static_cast< ::google::protobuf::uint32>( \ - reinterpret_cast( \ - &reinterpret_cast(16)->FIELD) - \ - reinterpret_cast(16)) -#endif - -// Constants for special floating point values. -LIBPROTOBUF_EXPORT double Infinity(); -LIBPROTOBUF_EXPORT double NaN(); -LIBPROTOBUF_EXPORT void InitProtobufDefaults(); +PROTOBUF_EXPORT void InitProtobufDefaults(); // This used by proto1 -inline const std::string& GetEmptyString() { +PROTOBUF_EXPORT inline const std::string& GetEmptyString() { InitProtobufDefaults(); return GetEmptyStringAlreadyInited(); } + // True if IsInitialized() is true for all elements of t. Type is expected // to be a RepeatedPtrField. It's useful to have this // helper here to keep the protobuf compiler from ever having to emit loops in // IsInitialized() methods. We want the C++ compiler to inline this or not // as it sees fit. -template bool AllAreInitialized(const Type& t) { - for (int i = t.size(); --i >= 0; ) { +template +bool AllAreInitialized(const RepeatedPtrField& t) { + for (int i = t.size(); --i >= 0;) { if (!t.Get(i).IsInitialized()) return false; } return true; @@ -133,10 +111,10 @@ template bool AllAreInitialized(const Type& t) { // This version operates on MessageLite to avoid introducing a dependency on the // concrete message type. template -bool AllAreInitializedWeak(const ::google::protobuf::RepeatedPtrField& t) { +bool AllAreInitializedWeak(const RepeatedPtrField& t) { for (int i = t.size(); --i >= 0;) { - if (!reinterpret_cast(t) - .Get<::google::protobuf::internal::ImplicitWeakTypeHandler >(i) + if (!reinterpret_cast(t) + .Get >(i) .IsInitialized()) { return false; } @@ -144,40 +122,6 @@ bool AllAreInitializedWeak(const ::google::protobuf::RepeatedPtrField& t) { return true; } -struct LIBPROTOBUF_EXPORT FieldMetadata { - uint32 offset; // offset of this field in the struct - uint32 tag; // field * 8 + wire_type - // byte offset * 8 + bit_offset; - // if the high bit is set then this is the byte offset of the oneof_case - // for this field. - uint32 has_offset; - uint32 type; // the type of this field. - const void* ptr; // auxiliary data - - // From the serializer point of view each fundamental type can occur in - // 4 different ways. For simplicity we treat all combinations as a cartesion - // product although not all combinations are allowed. - enum FieldTypeClass { - kPresence, - kNoPresence, - kRepeated, - kPacked, - kOneOf, - kNumTypeClasses // must be last enum - }; - // C++ protobuf has 20 fundamental types, were we added Cord and StringPiece - // and also distinquish the same types if they have different wire format. - enum { - kCordType = 19, - kStringPieceType = 20, - kInlinedType = 21, - kNumTypes = 21, - kSpecial = kNumTypes * kNumTypeClasses, - }; - - static int CalculateType(int fundamental_type, FieldTypeClass type_class); -}; - inline bool IsPresent(const void* base, uint32 hasbit) { const uint32* has_bits_array = static_cast(base); return (has_bits_array[hasbit / 32] & (1u << (hasbit & 31))) != 0; @@ -191,116 +135,23 @@ inline bool IsOneofPresent(const void* base, uint32 offset, uint32 tag) { typedef void (*SpecialSerializer)(const uint8* base, uint32 offset, uint32 tag, uint32 has_offset, - ::google::protobuf::io::CodedOutputStream* output); - -LIBPROTOBUF_EXPORT void ExtensionSerializer(const uint8* base, uint32 offset, uint32 tag, - uint32 has_offset, - ::google::protobuf::io::CodedOutputStream* output); -LIBPROTOBUF_EXPORT void UnknownFieldSerializerLite(const uint8* base, uint32 offset, uint32 tag, - uint32 has_offset, - ::google::protobuf::io::CodedOutputStream* output); - -struct SerializationTable { - int num_fields; - const FieldMetadata* field_table; -}; - -LIBPROTOBUF_EXPORT void SerializeInternal(const uint8* base, const FieldMetadata* table, - int num_fields, ::google::protobuf::io::CodedOutputStream* output); - -inline void TableSerialize(const ::google::protobuf::MessageLite& msg, - const SerializationTable* table, - ::google::protobuf::io::CodedOutputStream* output) { - const FieldMetadata* field_table = table->field_table; - int num_fields = table->num_fields - 1; - const uint8* base = reinterpret_cast(&msg); - // TODO(gerbens) This skips the first test if we could use the fast - // array serialization path, we should make this - // int cached_size = - // *reinterpret_cast(base + field_table->offset); - // SerializeWithCachedSize(msg, field_table + 1, num_fields, cached_size, ...) - // But we keep conformance with the old way for now. - SerializeInternal(base, field_table + 1, num_fields, output); -} - -uint8* SerializeInternalToArray(const uint8* base, const FieldMetadata* table, - int num_fields, bool is_deterministic, - uint8* buffer); - -inline uint8* TableSerializeToArray(const ::google::protobuf::MessageLite& msg, - const SerializationTable* table, - bool is_deterministic, uint8* buffer) { - const uint8* base = reinterpret_cast(&msg); - const FieldMetadata* field_table = table->field_table + 1; - int num_fields = table->num_fields - 1; - return SerializeInternalToArray(base, field_table, num_fields, - is_deterministic, buffer); -} - -template -struct CompareHelper { - bool operator()(const T& a, const T& b) { return a < b; } -}; - -template <> -struct CompareHelper { - bool operator()(const ArenaStringPtr& a, const ArenaStringPtr& b) { - return a.Get() < b.Get(); - } -}; - -struct CompareMapKey { - template - bool operator()(const MapEntryHelper& a, const MapEntryHelper& b) { - return Compare(a.key_, b.key_); - } - template - bool Compare(const T& a, const T& b) { - return CompareHelper()(a, b); - } -}; - -template -void MapFieldSerializer(const uint8* base, uint32 offset, uint32 tag, - uint32 has_offset, - ::google::protobuf::io::CodedOutputStream* output) { - typedef MapEntryHelper Entry; - typedef typename MapFieldType::MapType::const_iterator Iter; - - const MapFieldType& map_field = - *reinterpret_cast(base + offset); - const SerializationTable* t = - table + - has_offset; // has_offset is overloaded for maps to mean table offset - if (!output->IsSerializationDeterministic()) { - for (Iter it = map_field.GetMap().begin(); it != map_field.GetMap().end(); - ++it) { - Entry map_entry(*it); - output->WriteVarint32(tag); - output->WriteVarint32(map_entry._cached_size_); - SerializeInternal(reinterpret_cast(&map_entry), - t->field_table, t->num_fields, output); - } - } else { - std::vector v; - for (Iter it = map_field.GetMap().begin(); it != map_field.GetMap().end(); - ++it) { - v.push_back(Entry(*it)); - } - std::sort(v.begin(), v.end(), CompareMapKey()); - for (int i = 0; i < v.size(); i++) { - output->WriteVarint32(tag); - output->WriteVarint32(v[i]._cached_size_); - SerializeInternal(reinterpret_cast(&v[i]), t->field_table, - t->num_fields, output); - } - } -} - -LIBPROTOBUF_EXPORT MessageLite* DuplicateIfNonNullInternal(MessageLite* message); -LIBPROTOBUF_EXPORT MessageLite* GetOwnedMessageInternal(Arena* message_arena, - MessageLite* submessage, - Arena* submessage_arena); + io::CodedOutputStream* output); + +PROTOBUF_EXPORT void ExtensionSerializer(const uint8* base, uint32 offset, + uint32 tag, uint32 has_offset, + io::CodedOutputStream* output); +PROTOBUF_EXPORT void UnknownFieldSerializerLite(const uint8* base, + uint32 offset, uint32 tag, + uint32 has_offset, + io::CodedOutputStream* output); + +PROTOBUF_EXPORT MessageLite* DuplicateIfNonNullInternal(MessageLite* message); +PROTOBUF_EXPORT MessageLite* GetOwnedMessageInternal(Arena* message_arena, + MessageLite* submessage, + Arena* submessage_arena); +PROTOBUF_EXPORT void GenericSwap(MessageLite* m1, MessageLite* m2); +// We specialize GenericSwap for non-lite messages to benefit from reflection. +PROTOBUF_EXPORT void GenericSwap(Message* m1, Message* m2); template T* DuplicateIfNonNull(T* message) { @@ -322,17 +173,18 @@ T* GetOwnedMessage(Arena* message_arena, T* submessage, // Hide atomic from the public header and allow easy change to regular int // on platforms where the atomic might have a perf impact. -class LIBPROTOBUF_EXPORT CachedSize { +class PROTOBUF_EXPORT CachedSize { public: int Get() const { return size_.load(std::memory_order_relaxed); } void Set(int size) { size_.store(size, std::memory_order_relaxed); } + private: std::atomic size_{0}; }; // SCCInfo represents information of a strongly connected component of // mutual dependent messages. -struct LIBPROTOBUF_EXPORT SCCInfoBase { +struct PROTOBUF_EXPORT SCCInfoBase { // We use 0 for the Initialized state, because test eax,eax, jnz is smaller // and is subject to macro fusion. enum { @@ -340,52 +192,69 @@ struct LIBPROTOBUF_EXPORT SCCInfoBase { kRunning = 1, kUninitialized = -1, // initial state }; -#ifndef _MSC_VER - std::atomic visit_status; -#else - // MSVC doesnt make std::atomic constant initialized. This union trick +#if defined(_MSC_VER) && !defined(__clang__) + // MSVC doesn't make std::atomic constant initialized. This union trick // makes it so. union { int visit_status_to_make_linker_init; std::atomic visit_status; }; +#else + std::atomic visit_status; #endif int num_deps; + int num_implicit_weak_deps; void (*init_func)(); // This is followed by an array of num_deps // const SCCInfoBase* deps[]; }; +// Zero-length arrays are a language extension available in GCC and Clang but +// not MSVC. +#ifdef __GNUC__ +#define PROTOBUF_ARRAY_SIZE(n) (n) +#else +#define PROTOBUF_ARRAY_SIZE(n) ((n) ? (n) : 1) +#endif + template struct SCCInfo { SCCInfoBase base; // Semantically this is const SCCInfo* which is is a templated type. // The obvious inheriting from SCCInfoBase mucks with struct initialization. // Attempts showed the compiler was generating dynamic initialization code. - // Zero length arrays produce warnings with MSVC. - SCCInfoBase* deps[N ? N : 1]; + // This deps array consists of base.num_deps pointers to SCCInfoBase followed + // by base.num_implicit_weak_deps pointers to SCCInfoBase*. We need the extra + // pointer indirection for implicit weak fields. We cannot use a union type + // here, since that would prevent the array from being linker-initialized. + void* deps[PROTOBUF_ARRAY_SIZE(N)]; }; -LIBPROTOBUF_EXPORT void InitSCCImpl(SCCInfoBase* scc); +#undef PROTOBUF_ARRAY_SIZE + +PROTOBUF_EXPORT void InitSCCImpl(SCCInfoBase* scc); inline void InitSCC(SCCInfoBase* scc) { auto status = scc->visit_status.load(std::memory_order_acquire); - if (GOOGLE_PREDICT_FALSE(status != SCCInfoBase::kInitialized)) InitSCCImpl(scc); + if (PROTOBUF_PREDICT_FALSE(status != SCCInfoBase::kInitialized)) + InitSCCImpl(scc); } -LIBPROTOBUF_EXPORT void DestroyMessage(const void* message); -LIBPROTOBUF_EXPORT void DestroyString(const void* s); +PROTOBUF_EXPORT void DestroyMessage(const void* message); +PROTOBUF_EXPORT void DestroyString(const void* s); // Destroy (not delete) the message inline void OnShutdownDestroyMessage(const void* ptr) { OnShutdownRun(DestroyMessage, ptr); } -// Destroy the string (call string destructor) +// Destroy the string (call std::string destructor) inline void OnShutdownDestroyString(const std::string* ptr) { OnShutdownRun(DestroyString, ptr); } } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_GENERATED_MESSAGE_UTIL_H__ diff --git a/third_party/protobuf-lite/google/protobuf/has_bits.h b/third_party/protobuf-lite/google/protobuf/has_bits.h index e3a0149a..540cac28 100644 --- a/third_party/protobuf-lite/google/protobuf/has_bits.h +++ b/third_party/protobuf-lite/google/protobuf/has_bits.h @@ -32,27 +32,32 @@ #define GOOGLE_PROTOBUF_HAS_BITS_H__ #include -#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { namespace internal { -template +template class HasBits { public: - HasBits() GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { Clear(); } + HasBits() PROTOBUF_ALWAYS_INLINE { Clear(); } - void Clear() GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void Clear() PROTOBUF_ALWAYS_INLINE { memset(has_bits_, 0, sizeof(has_bits_)); } - ::google::protobuf::uint32& operator[](int index) GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + uint32& operator[](int index) PROTOBUF_ALWAYS_INLINE { return has_bits_[index]; } - const ::google::protobuf::uint32& operator[](int index) const - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + const uint32& operator[](int index) const PROTOBUF_ALWAYS_INLINE { return has_bits_[index]; } @@ -64,10 +69,14 @@ class HasBits { return !(*this == rhs); } + void Or(const HasBits& rhs) { + for (size_t i = 0; i < doublewords; i++) has_bits_[i] |= rhs[i]; + } + bool empty() const; private: - ::google::protobuf::uint32 has_bits_[doublewords]; + uint32 has_bits_[doublewords]; }; template <> @@ -100,6 +109,8 @@ inline bool HasBits::empty() const { } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_HAS_BITS_H__ diff --git a/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h b/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h index 3279bd17..ec028eb5 100644 --- a/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h +++ b/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h @@ -31,9 +31,18 @@ #ifndef GOOGLE_PROTOBUF_IMPLICIT_WEAK_MESSAGE_H__ #define GOOGLE_PROTOBUF_IMPLICIT_WEAK_MESSAGE_H__ +#include + #include #include #include +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + +#include // This file is logically internal-only and should only be used by protobuf // generated code. @@ -45,45 +54,44 @@ namespace internal { // An implementation of MessageLite that treats all data as unknown. This type // acts as a placeholder for an implicit weak field in the case where the true // message type does not get linked into the binary. -class LIBPROTOBUF_EXPORT ImplicitWeakMessage : public MessageLite { +class PROTOBUF_EXPORT ImplicitWeakMessage : public MessageLite { public: - ImplicitWeakMessage() : arena_(NULL) {} - explicit ImplicitWeakMessage(Arena* arena) : arena_(arena) {} + ImplicitWeakMessage() {} + explicit ImplicitWeakMessage(Arena* arena) : MessageLite(arena) {} static const ImplicitWeakMessage* default_instance(); - string GetTypeName() const { return ""; } + std::string GetTypeName() const override { return ""; } - MessageLite* New() const { return new ImplicitWeakMessage; } - MessageLite* New(Arena* arena) const { + MessageLite* New() const override { return new ImplicitWeakMessage; } + MessageLite* New(Arena* arena) const override { return Arena::CreateMessage(arena); } - Arena* GetArena() const { return arena_; } + void Clear() override { data_.clear(); } - void Clear() { data_.clear(); } + bool IsInitialized() const override { return true; } - bool IsInitialized() const { return true; } - - void CheckTypeAndMergeFrom(const MessageLite& other) { + void CheckTypeAndMergeFrom(const MessageLite& other) override { data_.append(static_cast(other).data_); } - bool MergePartialFromCodedStream(io::CodedInputStream* input); + const char* _InternalParse(const char* ptr, ParseContext* ctx) final; - size_t ByteSizeLong() const { return data_.size(); } + size_t ByteSizeLong() const override { return data_.size(); } - void SerializeWithCachedSizes(io::CodedOutputStream* output) const { - output->WriteString(data_); + uint8* _InternalSerialize(uint8* target, + io::EpsCopyOutputStream* stream) const final { + return stream->WriteRaw(data_.data(), static_cast(data_.size()), + target); } - int GetCachedSize() const { return static_cast(data_.size()); } + int GetCachedSize() const override { return static_cast(data_.size()); } typedef void InternalArenaConstructable_; private: - Arena* const arena_; - string data_; + std::string data_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ImplicitWeakMessage); }; @@ -91,45 +99,92 @@ class LIBPROTOBUF_EXPORT ImplicitWeakMessage : public MessageLite { template class ImplicitWeakTypeHandler { public: - typedef ImplicitWeakType Type; - typedef ::google::protobuf::MessageLite WeakType; - static const bool Moveable = false; - - // With implicit weak fields, we need separate NewFromPrototype and - // NewFromPrototypeWeak functions. The former is used when we want to create a - // strong dependency on the message type, and it just delegates to the - // GenericTypeHandler. The latter avoids creating a strong dependency, by - // simply calling MessageLite::New. - static inline ::google::protobuf::MessageLite* NewFromPrototype( - const ::google::protobuf::MessageLite* prototype, ::google::protobuf::Arena* arena = NULL) { + typedef MessageLite Type; + static constexpr bool Moveable = false; + + static inline MessageLite* NewFromPrototype(const MessageLite* prototype, + Arena* arena = NULL) { return prototype->New(arena); } - static inline void Delete(::google::protobuf::MessageLite* value, Arena* arena) { + static inline void Delete(MessageLite* value, Arena* arena) { if (arena == NULL) { delete value; } } - static inline ::google::protobuf::Arena* GetArena(::google::protobuf::MessageLite* value) { + static inline Arena* GetArena(MessageLite* value) { return value->GetArena(); } - static inline void* GetMaybeArenaPointer(::google::protobuf::MessageLite* value) { + static inline void* GetMaybeArenaPointer(MessageLite* value) { return value->GetArena(); } - static inline void Clear(::google::protobuf::MessageLite* value) { - value->Clear(); - } - static void Merge(const ::google::protobuf::MessageLite& from, - ::google::protobuf::MessageLite* to) { + static inline void Clear(MessageLite* value) { value->Clear(); } + static void Merge(const MessageLite& from, MessageLite* to) { to->CheckTypeAndMergeFrom(from); } - static inline size_t SpaceUsedLong(const Type& value) { - return value.SpaceUsedLong(); - } }; } // namespace internal -} // namespace protobuf +template +struct WeakRepeatedPtrField { + using TypeHandler = internal::ImplicitWeakTypeHandler; + WeakRepeatedPtrField() : weak() {} + explicit WeakRepeatedPtrField(Arena* arena) : weak(arena) {} + ~WeakRepeatedPtrField() { weak.template Destroy(); } + + typedef internal::RepeatedPtrIterator iterator; + typedef internal::RepeatedPtrIterator const_iterator; + typedef internal::RepeatedPtrOverPtrsIterator + pointer_iterator; + typedef internal::RepeatedPtrOverPtrsIterator + const_pointer_iterator; + + iterator begin() { return iterator(base().raw_data()); } + const_iterator begin() const { return iterator(base().raw_data()); } + const_iterator cbegin() const { return begin(); } + iterator end() { return begin() + base().size(); } + const_iterator end() const { return begin() + base().size(); } + const_iterator cend() const { return end(); } + pointer_iterator pointer_begin() { + return pointer_iterator(base().raw_mutable_data()); + } + const_pointer_iterator pointer_begin() const { + return const_pointer_iterator(base().raw_mutable_data()); + } + pointer_iterator pointer_end() { + return pointer_iterator(base().raw_mutable_data() + base().size()); + } + const_pointer_iterator pointer_end() const { + return const_pointer_iterator(base().raw_mutable_data() + base().size()); + } + + MessageLite* AddWeak(const MessageLite* prototype) { + return base().AddWeak(prototype); + } + T* Add() { return weak.Add(); } + void Clear() { base().template Clear(); } + void MergeFrom(const WeakRepeatedPtrField& other) { + base().template MergeFrom(other.base()); + } + void InternalSwap(WeakRepeatedPtrField* other) { + base().InternalSwap(&other->base()); + } + + const internal::RepeatedPtrFieldBase& base() const { return weak; } + internal::RepeatedPtrFieldBase& base() { return weak; } + // Union disables running the destructor. Which would create a strong link. + // Instead we explicitly destroy the underlying base through the virtual + // destructor. + union { + RepeatedPtrField weak; + }; +}; + +} // namespace protobuf } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_IMPLICIT_WEAK_MESSAGE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/inlined_string_field.h b/third_party/protobuf-lite/google/protobuf/inlined_string_field.h index 95d4687b..991c0e1f 100644 --- a/third_party/protobuf-lite/google/protobuf/inlined_string_field.h +++ b/third_party/protobuf-lite/google/protobuf/inlined_string_field.h @@ -32,9 +32,17 @@ #define GOOGLE_PROTOBUF_INLINED_STRING_FIELD_H__ #include +#include -#include -#include +#include +#include + +// Must be included last. +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { @@ -43,162 +51,143 @@ class Arena; namespace internal { -// InlinedStringField wraps a ::std::string instance and exposes an API similar to -// ArenaStringPtr's wrapping of a ::std::string* instance. As ::std::string is never -// allocated on the Arena, we expose only the *NoArena methods of +// InlinedStringField wraps a std::string instance and exposes an API similar to +// ArenaStringPtr's wrapping of a std::string* instance. As std::string is +// never allocated on the Arena, we expose only the *NoArena methods of // ArenaStringPtr. // // default_value parameters are taken for consistency with ArenaStringPtr, but // are not used for most methods. With inlining, these should be removed from // the generated binary. -class LIBPROTOBUF_EXPORT InlinedStringField { +class PROTOBUF_EXPORT InlinedStringField { public: - InlinedStringField() - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; - explicit InlinedStringField(const ::std::string& default_value); + InlinedStringField() PROTOBUF_ALWAYS_INLINE; + explicit InlinedStringField(const std::string& default_value); - void AssignWithDefault(const ::std::string* default_value, - const InlinedStringField& from) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; + void AssignWithDefault(const std::string* default_value, + const InlinedStringField& from) PROTOBUF_ALWAYS_INLINE; - void ClearToEmpty(const ::std::string* default_value, Arena* arena) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void ClearToEmpty(const std::string* default_value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { ClearToEmptyNoArena(default_value); } - void ClearNonDefaultToEmpty() GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void ClearNonDefaultToEmpty() PROTOBUF_ALWAYS_INLINE { ClearNonDefaultToEmptyNoArena(); } - void ClearToEmptyNoArena(const ::std::string* default_value) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void ClearToEmptyNoArena(const std::string* /*default_value*/) + PROTOBUF_ALWAYS_INLINE { ClearNonDefaultToEmptyNoArena(); } - void ClearNonDefaultToEmptyNoArena() - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; + void ClearNonDefaultToEmptyNoArena() PROTOBUF_ALWAYS_INLINE; - void ClearToDefault(const ::std::string* default_value, Arena* arena) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void ClearToDefault(const std::string* default_value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { ClearToDefaultNoArena(default_value); } - void ClearToDefaultNoArena(const ::std::string* default_value) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; + void ClearToDefaultNoArena(const std::string* default_value) + PROTOBUF_ALWAYS_INLINE; - void Destroy(const ::std::string* default_value, Arena* arena) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void Destroy(const std::string* default_value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { DestroyNoArena(default_value); } - void DestroyNoArena(const ::std::string* default_value) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; + void DestroyNoArena(const std::string* default_value) PROTOBUF_ALWAYS_INLINE; - const ::std::string& Get() const GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { - return GetNoArena(); - } - const ::std::string& GetNoArena() const GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; + const std::string& Get() const PROTOBUF_ALWAYS_INLINE { return GetNoArena(); } + const std::string& GetNoArena() const PROTOBUF_ALWAYS_INLINE; - ::std::string* Mutable(const ::std::string* default_value, Arena* arena) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + std::string* Mutable(const std::string* default_value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { return MutableNoArena(default_value); } - ::std::string* MutableNoArena(const ::std::string* default_value) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; + std::string* MutableNoArena(const std::string* default_value) + PROTOBUF_ALWAYS_INLINE; - ::std::string* Release(const ::std::string* default_value, Arena* arena) { + std::string* Release(const std::string* default_value, Arena* /*arena*/) { return ReleaseNoArena(default_value); } - ::std::string* ReleaseNonDefault(const ::std::string* default_value, Arena* arena) { + std::string* ReleaseNonDefault(const std::string* default_value, + Arena* /*arena*/) { return ReleaseNonDefaultNoArena(default_value); } - ::std::string* ReleaseNoArena(const ::std::string* default_value) { + std::string* ReleaseNoArena(const std::string* default_value) { return ReleaseNonDefaultNoArena(default_value); } - ::std::string* ReleaseNonDefaultNoArena(const ::std::string* default_value); + std::string* ReleaseNonDefaultNoArena(const std::string* default_value); - void Set(const ::std::string* default_value, - StringPiece value, - Arena* arena) GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void Set(const std::string* default_value, StringPiece value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { SetNoArena(default_value, value); } - void SetLite(const ::std::string* default_value, - StringPiece value, - Arena* arena) GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void SetLite(const std::string* default_value, StringPiece value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { SetNoArena(default_value, value); } - void SetNoArena(const ::std::string* default_value, - StringPiece value) GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; + void SetNoArena(const std::string* default_value, + StringPiece value) PROTOBUF_ALWAYS_INLINE; - void Set(const ::std::string* default_value, - const ::std::string& value, - Arena* arena) GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void Set(const std::string* default_value, const std::string& value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { SetNoArena(default_value, value); } - void SetLite(const ::std::string* default_value, - const ::std::string& value, - Arena* arena) GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE { + void SetLite(const std::string* default_value, const std::string& value, + Arena* /*arena*/) PROTOBUF_ALWAYS_INLINE { SetNoArena(default_value, value); } - void SetNoArena(const ::std::string* default_value, - const ::std::string& value) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; - -#if LANG_CXX11 - void SetNoArena(const ::std::string* default_value, - ::std::string&& value) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; -#endif - void SetAllocated(const ::std::string* default_value, - ::std::string* value, - Arena* arena) { + void SetNoArena(const std::string* default_value, + const std::string& value) PROTOBUF_ALWAYS_INLINE; + + void SetNoArena(const std::string* default_value, + std::string&& value) PROTOBUF_ALWAYS_INLINE; + void SetAllocated(const std::string* default_value, std::string* value, + Arena* /*arena*/) { SetAllocatedNoArena(default_value, value); } - void SetAllocatedNoArena(const ::std::string* default_value, - ::std::string* value); - void Swap(InlinedStringField* from) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE; - ::std::string* UnsafeMutablePointer(); - void UnsafeSetDefault(const ::std::string* default_value); - ::std::string* UnsafeArenaRelease(const ::std::string* default_value, Arena* arena); - void UnsafeArenaSetAllocated( - const ::std::string* default_value, ::std::string* value, Arena* arena); - - bool IsDefault(const ::std::string* default_value) { - return false; - } + void SetAllocatedNoArena(const std::string* default_value, + std::string* value); + void Swap(InlinedStringField* from) PROTOBUF_ALWAYS_INLINE; + std::string* UnsafeMutablePointer(); + void UnsafeSetDefault(const std::string* default_value); + std::string* UnsafeArenaRelease(const std::string* default_value, + Arena* arena); + void UnsafeArenaSetAllocated(const std::string* default_value, + std::string* value, Arena* arena); + + bool IsDefault(const std::string* /*default_value*/) { return false; } + private: - ::std::string value_; + std::string value_; }; inline InlinedStringField::InlinedStringField() {} -inline InlinedStringField::InlinedStringField(const ::std::string& default_value) : - value_(default_value) {} +inline InlinedStringField::InlinedStringField(const std::string& default_value) + : value_(default_value) {} inline void InlinedStringField::AssignWithDefault( - const ::std::string* default_value, const InlinedStringField& from) { + const std::string* /*default_value*/, const InlinedStringField& from) { value_ = from.value_; } -inline const ::std::string& InlinedStringField::GetNoArena() const { +inline const std::string& InlinedStringField::GetNoArena() const { return value_; } -inline ::std::string* InlinedStringField::MutableNoArena(const ::std::string*) { +inline std::string* InlinedStringField::MutableNoArena(const std::string*) { return &value_; } inline void InlinedStringField::SetAllocatedNoArena( - const ::std::string* default_value, ::std::string* value) { + const std::string* default_value, std::string* value) { if (value == NULL) { value_.assign(*default_value); } else { -#if LANG_CXX11 value_.assign(std::move(*value)); -#else - value_.swap(*value); -#endif delete value; } } -inline void InlinedStringField::DestroyNoArena(const ::std::string*) { +inline void InlinedStringField::DestroyNoArena(const std::string*) { // This is invoked from the generated message's ArenaDtor, which is used to // clean up objects not allocated on the Arena. this->~InlinedStringField(); @@ -209,54 +198,52 @@ inline void InlinedStringField::ClearNonDefaultToEmptyNoArena() { } inline void InlinedStringField::ClearToDefaultNoArena( - const ::std::string* default_value) { + const std::string* default_value) { value_.assign(*default_value); } -inline ::std::string* InlinedStringField::ReleaseNonDefaultNoArena( - const ::std::string* default_value) { - ::std::string* released = new ::std::string(*default_value); +inline std::string* InlinedStringField::ReleaseNonDefaultNoArena( + const std::string* default_value) { + std::string* released = new std::string(*default_value); value_.swap(*released); return released; } -inline void InlinedStringField::SetNoArena( - const ::std::string* default_value, StringPiece value) { +inline void InlinedStringField::SetNoArena(const std::string* /*default_value*/, + StringPiece value) { value_.assign(value.data(), value.length()); } -inline void InlinedStringField::SetNoArena( - const ::std::string* default_value, const ::std::string& value) { +inline void InlinedStringField::SetNoArena(const std::string* /*default_value*/, + const std::string& value) { value_.assign(value); } -#if LANG_CXX11 -inline void InlinedStringField::SetNoArena( - const ::std::string* default_value, ::std::string&& value) { +inline void InlinedStringField::SetNoArena(const std::string* /*default_value*/, + std::string&& value) { value_.assign(std::move(value)); } -#endif inline void InlinedStringField::Swap(InlinedStringField* from) { value_.swap(from->value_); } -inline ::std::string* InlinedStringField::UnsafeMutablePointer() { +inline std::string* InlinedStringField::UnsafeMutablePointer() { return &value_; } inline void InlinedStringField::UnsafeSetDefault( - const ::std::string* default_value) { + const std::string* default_value) { value_.assign(*default_value); } -inline ::std::string* InlinedStringField::UnsafeArenaRelease( - const ::std::string* default_value, Arena* arena) { +inline std::string* InlinedStringField::UnsafeArenaRelease( + const std::string* default_value, Arena* /*arena*/) { return ReleaseNoArena(default_value); } inline void InlinedStringField::UnsafeArenaSetAllocated( - const ::std::string* default_value, ::std::string* value, Arena* arena) { + const std::string* default_value, std::string* value, Arena* /*arena*/) { if (value == NULL) { value_.assign(*default_value); } else { @@ -266,6 +253,8 @@ inline void InlinedStringField::UnsafeArenaSetAllocated( } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_INLINED_STRING_FIELD_H__ diff --git a/third_party/protobuf-lite/google/protobuf/io/coded_stream.h b/third_party/protobuf-lite/google/protobuf/io/coded_stream.h index 0f70ecde..5f9feb80 100644 --- a/third_party/protobuf-lite/google/protobuf/io/coded_stream.h +++ b/third_party/protobuf-lite/google/protobuf/io/coded_stream.h @@ -67,7 +67,7 @@ // // Read a file created by the above code. // int fd = open("myfile", O_RDONLY); // ZeroCopyInputStream* raw_input = new FileInputStream(fd); -// CodedInputStream coded_input = new CodedInputStream(raw_input); +// CodedInputStream* coded_input = new CodedInputStream(raw_input); // // coded_input->ReadLittleEndian32(&magic_number); // if (magic_number != 1234) { @@ -109,41 +109,55 @@ #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ + #include + #include #include +#include +#include #include +#include #include + #ifdef _MSC_VER - // Assuming windows is always little-endian. - #if !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) - #define PROTOBUF_LITTLE_ENDIAN 1 - #endif - #if _MSC_VER >= 1300 && !defined(__INTEL_COMPILER) - // If MSVC has "/RTCc" set, it will complain about truncating casts at - // runtime. This file contains some intentional truncating casts. - #pragma runtime_checks("c", off) - #endif +// Assuming windows is always little-endian. +#if !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) +#define PROTOBUF_LITTLE_ENDIAN 1 +#endif +#if _MSC_VER >= 1300 && !defined(__INTEL_COMPILER) +// If MSVC has "/RTCc" set, it will complain about truncating casts at +// runtime. This file contains some intentional truncating casts. +#pragma runtime_checks("c", off) +#endif #else - #include // __BYTE_ORDER - #if ((defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)) || \ - (defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN)) && \ - !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) - #define PROTOBUF_LITTLE_ENDIAN 1 - #endif +#include // __BYTE_ORDER +#if ((defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)) || \ + (defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN)) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) +#define PROTOBUF_LITTLE_ENDIAN 1 +#endif #endif #include -#include +#include +#include +#include #include -namespace google { +#include + +namespace google { namespace protobuf { class DescriptorPool; class MessageFactory; +class ZeroCopyCodedInputStream; -namespace internal { void MapTestForceDeterministic(); } +namespace internal { +void MapTestForceDeterministic(); +class EpsCopyByteStream; +} // namespace internal namespace io { @@ -152,8 +166,8 @@ class CodedInputStream; class CodedOutputStream; // Defined in other files. -class ZeroCopyInputStream; // zero_copy_stream.h -class ZeroCopyOutputStream; // zero_copy_stream.h +class ZeroCopyInputStream; // zero_copy_stream.h +class ZeroCopyOutputStream; // zero_copy_stream.h // Class which reads and decodes binary data which is composed of varint- // encoded integers and fixed-width pieces. Wraps a ZeroCopyInputStream. @@ -162,7 +176,9 @@ class ZeroCopyOutputStream; // zero_copy_stream.h // Most methods of CodedInputStream that return a bool return false if an // underlying I/O error occurs or if the data is malformed. Once such a // failure occurs, the CodedInputStream is broken and is no longer useful. -class LIBPROTOBUF_EXPORT CodedInputStream { +// After a failure, callers also should assume writes to "out" args may have +// occurred, though nothing useful can be determined from those writes. +class PROTOBUF_EXPORT CodedInputStream { public: // Create a CodedInputStream that reads from the given ZeroCopyInputStream. explicit CodedInputStream(ZeroCopyInputStream* input); @@ -198,23 +214,14 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // Like GetDirectBufferPointer, but this method is inlined, and does not // attempt to Refresh() if the buffer is currently empty. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE + PROTOBUF_ALWAYS_INLINE void GetDirectBufferPointerInline(const void** data, int* size); // Read raw bytes, copying them into the given buffer. bool ReadRaw(void* buffer, int size); - // Like the above, with inlined optimizations. This should only be used - // by the protobuf implementation. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - bool InternalReadRawInline(void* buffer, int size); - // Like ReadRaw, but reads into a string. - bool ReadString(string* buffer, int size); - // Like the above, with inlined optimizations. This should only be used - // by the protobuf implementation. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - bool InternalReadStringInline(string* buffer, int size); + bool ReadString(std::string* buffer, int size); // Read a 32-bit little-endian integer. @@ -226,10 +233,10 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // responsible for ensuring that the buffer has sufficient space. // Read a 32-bit little-endian integer. static const uint8* ReadLittleEndian32FromArray(const uint8* buffer, - uint32* value); + uint32* value); // Read a 64-bit little-endian integer. static const uint8* ReadLittleEndian64FromArray(const uint8* buffer, - uint64* value); + uint64* value); // Read an unsigned integer with Varint encoding, truncating to 32 bits. // Reading a 32-bit value is equivalent to reading a 64-bit one and casting @@ -257,12 +264,11 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // Always inline because this is only called in one place per parse loop // but it is called for every iteration of said loop, so it should be fast. // GCC doesn't want to inline this by default. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTag() { + PROTOBUF_ALWAYS_INLINE uint32 ReadTag() { return last_tag_ = ReadTagNoLastTag(); } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTagNoLastTag(); - + PROTOBUF_ALWAYS_INLINE uint32 ReadTagNoLastTag(); // This usually a faster alternative to ReadTag() when cutoff is a manifest // constant. It does particularly well for cutoff >= 127. The first part @@ -272,14 +278,14 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // above cutoff or is 0. (There's intentional wiggle room when tag is 0, // because that can arise in several ways, and for best performance we want // to avoid an extra "is tag == 0?" check here.) - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE + PROTOBUF_ALWAYS_INLINE std::pair ReadTagWithCutoff(uint32 cutoff) { std::pair result = ReadTagWithCutoffNoLastTag(cutoff); last_tag_ = result.first; return result; } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE + PROTOBUF_ALWAYS_INLINE std::pair ReadTagWithCutoffNoLastTag(uint32 cutoff); // Usually returns true if calling ReadVarint32() now would produce the given @@ -289,7 +295,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // parameter. // Always inline because this collapses to a small number of instructions // when given a constant parameter, but GCC doesn't want to inline by default. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE bool ExpectTag(uint32 expected); + PROTOBUF_ALWAYS_INLINE bool ExpectTag(uint32 expected); // Like above, except this reads from the specified buffer. The caller is // responsible for ensuring that the buffer is large enough to read a varint @@ -298,7 +304,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // // Returns a pointer beyond the expected tag if it was found, or NULL if it // was not. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE + PROTOBUF_ALWAYS_INLINE static const uint8* ExpectTagFromArray(const uint8* buffer, uint32 expected); // Usually returns true if no more bytes can be read. Always returns false @@ -328,6 +334,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // It also checks for some cases where, due to optimizations, // MergeFromCodedStream() can incorrectly return true. bool ConsumedEntireMessage(); + void SetConsumed() { legitimate_message_end_ = true; } // Limits ---------------------------------------------------------- // Limits are used when parsing length-delimited embedded messages. @@ -384,7 +391,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // This is unrelated to PushLimit()/PopLimit(). void SetTotalBytesLimit(int total_bytes_limit); - PROTOBUF_RUNTIME_DEPRECATED( + PROTOBUF_DEPRECATED_MSG( "Please use the single parameter version of SetTotalBytesLimit(). The " "second parameter is ignored.") void SetTotalBytesLimit(int total_bytes_limit, int) { @@ -403,7 +410,9 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // Sets the maximum recursion depth. The default is 100. void SetRecursionLimit(int limit); + int RecursionBudget() { return recursion_budget_; } + static int GetDefaultRecursionLimit() { return default_recursion_limit_; } // Increments the current recursion depth. Returns true if the depth is // under the limit, false if it has gone over. @@ -527,7 +536,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedInputStream); const uint8* buffer_; - const uint8* buffer_end_; // pointer to the end of the buffer. + const uint8* buffer_end_; // pointer to the end of the buffer. ZeroCopyInputStream* input_; int total_bytes_read_; // total bytes read from input_, including // the current buffer @@ -537,7 +546,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { int overflow_bytes_; // LastTagWas() stuff. - uint32 last_tag_; // result of last ReadTag() or ReadTagWithCutoff(). + uint32 last_tag_; // result of last ReadTag() or ReadTagWithCutoff(). // This is set true by ReadTag{Fallback/Slow}() if it is called when exactly // at EOF, or by ExpectAtEnd() when it returns true. This happens when we @@ -548,7 +557,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { bool aliasing_enabled_; // Limits - Limit current_limit_; // if position = -1, no limit is applied + Limit current_limit_; // if position = -1, no limit is applied // For simplicity, if the current buffer crosses a limit (either a normal // limit created by PushLimit() or the total bytes limit), buffer_size_ @@ -621,7 +630,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // stream. uint32 ReadTagFallback(uint32 first_byte_or_zero); uint32 ReadTagSlow(); - bool ReadStringFallback(string* buffer, int size); + bool ReadStringFallback(std::string* buffer, int size); // Return the size of the buffer. int BufferSize() const; @@ -629,8 +638,365 @@ class LIBPROTOBUF_EXPORT CodedInputStream { static const int kDefaultTotalBytesLimit = INT_MAX; static int default_recursion_limit_; // 100 by default. + + friend class google::protobuf::ZeroCopyCodedInputStream; + friend class google::protobuf::internal::EpsCopyByteStream; +}; + +// EpsCopyOutputStream wraps a ZeroCopyOutputStream and exposes a new stream, +// which has the property you can write kSlopBytes (16 bytes) from the current +// position without bounds checks. The cursor into the stream is managed by +// the user of the class and is an explicit parameter in the methods. Careful +// use of this class, ie. keep ptr a local variable, eliminates the need to +// for the compiler to sync the ptr value between register and memory. +class PROTOBUF_EXPORT EpsCopyOutputStream { + public: + enum { kSlopBytes = 16 }; + + // Initialize from a stream. + EpsCopyOutputStream(ZeroCopyOutputStream* stream, bool deterministic, + uint8** pp) + : end_(buffer_), + stream_(stream), + is_serialization_deterministic_(deterministic) { + *pp = buffer_; + } + + // Only for array serialization. No overflow protection, end_ will be the + // pointed to the end of the array. When using this the total size is already + // known, so no need to maintain the slop region. + EpsCopyOutputStream(void* data, int size, bool deterministic) + : end_(static_cast(data) + size), + buffer_end_(nullptr), + stream_(nullptr), + is_serialization_deterministic_(deterministic) {} + + // Initialize from stream but with the first buffer already given (eager). + EpsCopyOutputStream(void* data, int size, ZeroCopyOutputStream* stream, + bool deterministic, uint8** pp) + : stream_(stream), is_serialization_deterministic_(deterministic) { + *pp = SetInitialBuffer(data, size); + } + + // Flush everything that's written into the underlying ZeroCopyOutputStream + // and trims the underlying stream to the location of ptr. + uint8* Trim(uint8* ptr); + + // After this it's guaranteed you can safely write kSlopBytes to ptr. This + // will never fail! The underlying stream can produce an error. Use HadError + // to check for errors. + PROTOBUF_MUST_USE_RESULT uint8* EnsureSpace(uint8* ptr) { + if (PROTOBUF_PREDICT_FALSE(ptr >= end_)) { + return EnsureSpaceFallback(ptr); + } + return ptr; + } + + uint8* WriteRaw(const void* data, int size, uint8* ptr) { + if (PROTOBUF_PREDICT_FALSE(end_ - ptr < size)) { + return WriteRawFallback(data, size, ptr); + } + std::memcpy(ptr, data, size); + return ptr + size; + } + // Writes the buffer specified by data, size to the stream. Possibly by + // aliasing the buffer (ie. not copying the data). The caller is responsible + // to make sure the buffer is alive for the duration of the + // ZeroCopyOutputStream. + uint8* WriteRawMaybeAliased(const void* data, int size, uint8* ptr) { + if (aliasing_enabled_) { + return WriteAliasedRaw(data, size, ptr); + } else { + return WriteRaw(data, size, ptr); + } + } + + + uint8* WriteStringMaybeAliased(uint32 num, const std::string& s, uint8* ptr) { + std::ptrdiff_t size = s.size(); + if (PROTOBUF_PREDICT_FALSE( + size >= 128 || end_ - ptr + 16 - TagSize(num << 3) - 1 < size)) { + return WriteStringMaybeAliasedOutline(num, s, ptr); + } + ptr = UnsafeVarint((num << 3) | 2, ptr); + *ptr++ = static_cast(size); + std::memcpy(ptr, s.data(), size); + return ptr + size; + } + uint8* WriteBytesMaybeAliased(uint32 num, const std::string& s, uint8* ptr) { + return WriteStringMaybeAliased(num, s, ptr); + } + + template + PROTOBUF_ALWAYS_INLINE uint8* WriteString(uint32 num, const T& s, + uint8* ptr) { + std::ptrdiff_t size = s.size(); + if (PROTOBUF_PREDICT_FALSE( + size >= 128 || end_ - ptr + 16 - TagSize(num << 3) - 1 < size)) { + return WriteStringOutline(num, s, ptr); + } + ptr = UnsafeVarint((num << 3) | 2, ptr); + *ptr++ = static_cast(size); + std::memcpy(ptr, s.data(), size); + return ptr + size; + } + template + uint8* WriteBytes(uint32 num, const T& s, uint8* ptr) { + return WriteString(num, s, ptr); + } + + template + PROTOBUF_ALWAYS_INLINE uint8* WriteInt32Packed(int num, const T& r, int size, + uint8* ptr) { + return WriteVarintPacked(num, r, size, ptr, Encode64); + } + template + PROTOBUF_ALWAYS_INLINE uint8* WriteUInt32Packed(int num, const T& r, int size, + uint8* ptr) { + return WriteVarintPacked(num, r, size, ptr, Encode32); + } + template + PROTOBUF_ALWAYS_INLINE uint8* WriteSInt32Packed(int num, const T& r, int size, + uint8* ptr) { + return WriteVarintPacked(num, r, size, ptr, ZigZagEncode32); + } + template + PROTOBUF_ALWAYS_INLINE uint8* WriteInt64Packed(int num, const T& r, int size, + uint8* ptr) { + return WriteVarintPacked(num, r, size, ptr, Encode64); + } + template + PROTOBUF_ALWAYS_INLINE uint8* WriteUInt64Packed(int num, const T& r, int size, + uint8* ptr) { + return WriteVarintPacked(num, r, size, ptr, Encode64); + } + template + PROTOBUF_ALWAYS_INLINE uint8* WriteSInt64Packed(int num, const T& r, int size, + uint8* ptr) { + return WriteVarintPacked(num, r, size, ptr, ZigZagEncode64); + } + template + PROTOBUF_ALWAYS_INLINE uint8* WriteEnumPacked(int num, const T& r, int size, + uint8* ptr) { + return WriteVarintPacked(num, r, size, ptr, Encode64); + } + + template + PROTOBUF_ALWAYS_INLINE uint8* WriteFixedPacked(int num, const T& r, + uint8* ptr) { + ptr = EnsureSpace(ptr); + constexpr auto element_size = sizeof(typename T::value_type); + auto size = r.size() * element_size; + ptr = WriteLengthDelim(num, size, ptr); + return WriteRawLittleEndian(r.data(), static_cast(size), + ptr); + } + + // Returns true if there was an underlying I/O error since this object was + // created. + bool HadError() const { return had_error_; } + + // Instructs the EpsCopyOutputStream to allow the underlying + // ZeroCopyOutputStream to hold pointers to the original structure instead of + // copying, if it supports it (i.e. output->AllowsAliasing() is true). If the + // underlying stream does not support aliasing, then enabling it has no + // affect. For now, this only affects the behavior of + // WriteRawMaybeAliased(). + // + // NOTE: It is caller's responsibility to ensure that the chunk of memory + // remains live until all of the data has been consumed from the stream. + void EnableAliasing(bool enabled); + + // See documentation on CodedOutputStream::SetSerializationDeterministic. + void SetSerializationDeterministic(bool value) { + is_serialization_deterministic_ = value; + } + + // See documentation on CodedOutputStream::IsSerializationDeterministic. + bool IsSerializationDeterministic() const { + return is_serialization_deterministic_; + } + + // The number of bytes written to the stream at position ptr, relative to the + // stream's overall position. + int64 ByteCount(uint8* ptr) const; + + + private: + uint8* end_; + uint8* buffer_end_ = buffer_; + uint8 buffer_[2 * kSlopBytes]; + ZeroCopyOutputStream* stream_; + bool had_error_ = false; + bool aliasing_enabled_ = false; // See EnableAliasing(). + bool is_serialization_deterministic_; + + uint8* EnsureSpaceFallback(uint8* ptr); + inline uint8* Next(); + int Flush(uint8* ptr); + std::ptrdiff_t GetSize(uint8* ptr) const { + GOOGLE_DCHECK(ptr <= end_ + kSlopBytes); // NOLINT + return end_ + kSlopBytes - ptr; + } + + uint8* Error() { + had_error_ = true; + // We use the patch buffer to always guarantee space to write to. + end_ = buffer_ + kSlopBytes; + return buffer_; + } + + static constexpr int TagSize(uint32 tag) { + return (tag < (1 << 7)) + ? 1 + : (tag < (1 << 14)) + ? 2 + : (tag < (1 << 21)) ? 3 : (tag < (1 << 28)) ? 4 : 5; + } + + PROTOBUF_ALWAYS_INLINE uint8* WriteTag(uint32 num, uint32 wt, uint8* ptr) { + GOOGLE_DCHECK(ptr < end_); // NOLINT + return UnsafeVarint((num << 3) | wt, ptr); + } + + PROTOBUF_ALWAYS_INLINE uint8* WriteLengthDelim(int num, uint32 size, + uint8* ptr) { + ptr = WriteTag(num, 2, ptr); + return UnsafeWriteSize(size, ptr); + } + + uint8* WriteRawFallback(const void* data, int size, uint8* ptr); + + uint8* WriteAliasedRaw(const void* data, int size, uint8* ptr); + + uint8* WriteStringMaybeAliasedOutline(uint32 num, const std::string& s, + uint8* ptr); + uint8* WriteStringOutline(uint32 num, const std::string& s, uint8* ptr); + + template + PROTOBUF_ALWAYS_INLINE uint8* WriteVarintPacked(int num, const T& r, int size, + uint8* ptr, const E& encode) { + ptr = EnsureSpace(ptr); + ptr = WriteLengthDelim(num, size, ptr); + auto it = r.data(); + auto end = it + r.size(); + do { + ptr = EnsureSpace(ptr); + ptr = UnsafeVarint(encode(*it++), ptr); + } while (it < end); + return ptr; + } + + static uint32 Encode32(uint32 v) { return v; } + static uint64 Encode64(uint64 v) { return v; } + static uint32 ZigZagEncode32(int32 v) { + return (static_cast(v) << 1) ^ static_cast(v >> 31); + } + static uint64 ZigZagEncode64(int64 v) { + return (static_cast(v) << 1) ^ static_cast(v >> 63); + } + + template + PROTOBUF_ALWAYS_INLINE static uint8* UnsafeVarint(T value, uint8* ptr) { + static_assert(std::is_unsigned::value, + "Varint serialization must be unsigned"); + if (value < 0x80) { + ptr[0] = static_cast(value); + return ptr + 1; + } + ptr[0] = static_cast(value | 0x80); + value >>= 7; + if (value < 0x80) { + ptr[1] = static_cast(value); + return ptr + 2; + } + ptr++; + do { + *ptr = static_cast(value | 0x80); + value >>= 7; + ++ptr; + } while (PROTOBUF_PREDICT_FALSE(value >= 0x80)); + *ptr++ = static_cast(value); + return ptr; + } + + PROTOBUF_ALWAYS_INLINE static uint8* UnsafeWriteSize(uint32 value, + uint8* ptr) { + while (PROTOBUF_PREDICT_FALSE(value >= 0x80)) { + *ptr = static_cast(value | 0x80); + value >>= 7; + ++ptr; + } + *ptr++ = static_cast(value); + return ptr; + } + + template + uint8* WriteRawLittleEndian(const void* data, int size, uint8* ptr); +#ifndef PROTOBUF_LITTLE_ENDIAN + uint8* WriteRawLittleEndian32(const void* data, int size, uint8* ptr); + uint8* WriteRawLittleEndian64(const void* data, int size, uint8* ptr); +#endif + + // These methods are for CodedOutputStream. Ideally they should be private + // but to match current behavior of CodedOutputStream as close as possible + // we allow it some functionality. + public: + uint8* SetInitialBuffer(void* data, int size) { + auto ptr = static_cast(data); + if (size > kSlopBytes) { + end_ = ptr + size - kSlopBytes; + buffer_end_ = nullptr; + return ptr; + } else { + end_ = buffer_ + size; + buffer_end_ = ptr; + return buffer_; + } + } + + private: + // Needed by CodedOutputStream HadError. HadError needs to flush the patch + // buffers to ensure there is no error as of yet. + uint8* FlushAndResetBuffer(uint8*); + + // The following functions mimick the old CodedOutputStream behavior as close + // as possible. They flush the current state to the stream, behave as + // the old CodedOutputStream and then return to normal operation. + bool Skip(int count, uint8** pp); + bool GetDirectBufferPointer(void** data, int* size, uint8** pp); + uint8* GetDirectBufferForNBytesAndAdvance(int size, uint8** pp); + + friend class CodedOutputStream; }; +template <> +inline uint8* EpsCopyOutputStream::WriteRawLittleEndian<1>(const void* data, + int size, + uint8* ptr) { + return WriteRaw(data, size, ptr); +} +template <> +inline uint8* EpsCopyOutputStream::WriteRawLittleEndian<4>(const void* data, + int size, + uint8* ptr) { +#ifdef PROTOBUF_LITTLE_ENDIAN + return WriteRaw(data, size, ptr); +#else + return WriteRawLittleEndian32(data, size, ptr); +#endif +} +template <> +inline uint8* EpsCopyOutputStream::WriteRawLittleEndian<8>(const void* data, + int size, + uint8* ptr) { +#ifdef PROTOBUF_LITTLE_ENDIAN + return WriteRaw(data, size, ptr); +#else + return WriteRawLittleEndian64(data, size, ptr); +#endif +} + // Class which encodes and writes binary data which is composed of varint- // encoded integers and fixed-width pieces. Wraps a ZeroCopyOutputStream. // Most users will not need to deal with CodedOutputStream. @@ -651,7 +1017,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // individual value. // i.e., in the example above: // -// CodedOutputStream coded_output = new CodedOutputStream(raw_output); +// CodedOutputStream* coded_output = new CodedOutputStream(raw_output); // int magic_number = 1234; // char text[] = "Hello world!"; // @@ -661,7 +1027,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // // uint8* buffer = // coded_output->GetDirectBufferForNBytesAndAdvance(coded_size); -// if (buffer != NULL) { +// if (buffer != nullptr) { // // The output stream has enough space in the buffer: write directly to // // the array. // buffer = CodedOutputStream::WriteLittleEndian32ToArray(magic_number, @@ -677,27 +1043,40 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // } // // delete coded_output; -class LIBPROTOBUF_EXPORT CodedOutputStream { +class PROTOBUF_EXPORT CodedOutputStream { public: // Create an CodedOutputStream that writes to the given ZeroCopyOutputStream. - explicit CodedOutputStream(ZeroCopyOutputStream* output); - CodedOutputStream(ZeroCopyOutputStream* output, bool do_eager_refresh); + explicit CodedOutputStream(ZeroCopyOutputStream* stream) + : CodedOutputStream(stream, true) {} + CodedOutputStream(ZeroCopyOutputStream* stream, bool do_eager_refresh); // Destroy the CodedOutputStream and position the underlying // ZeroCopyOutputStream immediately after the last byte written. ~CodedOutputStream(); + // Returns true if there was an underlying I/O error since this object was + // created. On should call Trim before this function in order to catch all + // errors. + bool HadError() { + cur_ = impl_.FlushAndResetBuffer(cur_); + GOOGLE_DCHECK(cur_); + return impl_.HadError(); + } + // Trims any unused space in the underlying buffer so that its size matches // the number of bytes written by this stream. The underlying buffer will // automatically be trimmed when this stream is destroyed; this call is only // necessary if the underlying buffer is accessed *before* the stream is // destroyed. - void Trim(); + void Trim() { cur_ = impl_.Trim(cur_); } // Skips a number of bytes, leaving the bytes unmodified in the underlying // buffer. Returns false if an underlying write error occurs. This is // mainly useful with GetDirectBufferPointer(). - bool Skip(int count); + // Note of caution, the skipped bytes may contain uninitialized data. The + // caller must make sure that the skipped bytes are properly initialized, + // otherwise you might leak bytes from your heap. + bool Skip(int count) { return impl_.Skip(count, &cur_); } // Sets *data to point directly at the unwritten part of the // CodedOutputStream's underlying buffer, and *size to the size of that @@ -707,7 +1086,9 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // the consumed bytes. This may be useful for implementing external fast // serialization routines for types of data not covered by the // CodedOutputStream interface. - bool GetDirectBufferPointer(void** data, int* size); + bool GetDirectBufferPointer(void** data, int* size) { + return impl_.GetDirectBufferPointer(data, size, &cur_); + } // If there are at least "size" bytes available in the current buffer, // returns a pointer directly into the buffer and advances over these bytes. @@ -716,10 +1097,14 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // there are not enough bytes available, returns NULL. The return pointer is // invalidated as soon as any other non-const method of CodedOutputStream // is called. - inline uint8* GetDirectBufferForNBytesAndAdvance(int size); + inline uint8* GetDirectBufferForNBytesAndAdvance(int size) { + return impl_.GetDirectBufferForNBytesAndAdvance(size, &cur_); + } // Write raw bytes, copying them from the given buffer. - void WriteRaw(const void* buffer, int size); + void WriteRaw(const void* buffer, int size) { + cur_ = impl_.WriteRaw(buffer, size, cur_); + } // Like WriteRaw() but will try to write aliased data if aliasing is // turned on. void WriteRawMaybeAliased(const void* data, int size); @@ -731,30 +1116,26 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { static uint8* WriteRawToArray(const void* buffer, int size, uint8* target); // Equivalent to WriteRaw(str.data(), str.size()). - void WriteString(const string& str); + void WriteString(const std::string& str); // Like WriteString() but writing directly to the target array. - static uint8* WriteStringToArray(const string& str, uint8* target); + static uint8* WriteStringToArray(const std::string& str, uint8* target); // Write the varint-encoded size of str followed by str. - static uint8* WriteStringWithSizeToArray(const string& str, uint8* target); + static uint8* WriteStringWithSizeToArray(const std::string& str, + uint8* target); - // Instructs the CodedOutputStream to allow the underlying - // ZeroCopyOutputStream to hold pointers to the original structure instead of - // copying, if it supports it (i.e. output->AllowsAliasing() is true). If the - // underlying stream does not support aliasing, then enabling it has no - // affect. For now, this only affects the behavior of - // WriteRawMaybeAliased(). - // - // NOTE: It is caller's responsibility to ensure that the chunk of memory - // remains live until all of the data has been consumed from the stream. - void EnableAliasing(bool enabled); - // Write a 32-bit little-endian integer. - void WriteLittleEndian32(uint32 value); + void WriteLittleEndian32(uint32 value) { + cur_ = impl_.EnsureSpace(cur_); + SetCur(WriteLittleEndian32ToArray(value, Cur())); + } // Like WriteLittleEndian32() but writing directly to the target array. static uint8* WriteLittleEndian32ToArray(uint32 value, uint8* target); // Write a 64-bit little-endian integer. - void WriteLittleEndian64(uint64 value); + void WriteLittleEndian64(uint64 value) { + cur_ = impl_.EnsureSpace(cur_); + SetCur(WriteLittleEndian64ToArray(value, Cur())); + } // Like WriteLittleEndian64() but writing directly to the target array. static uint8* WriteLittleEndian64ToArray(uint64 value, uint8* target); @@ -782,7 +1163,7 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // but GCC by default doesn't want to inline this. void WriteTag(uint32 value); // Like WriteTag() but writing directly to the target array. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE + PROTOBUF_ALWAYS_INLINE static uint8* WriteTagToArray(uint32 value, uint8* target); // Returns the number of bytes needed to encode the given value as a varint. @@ -790,7 +1171,7 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // Returns the number of bytes needed to encode the given value as a varint. static size_t VarintSize64(uint64 value); - // If negative, 10 bytes. Otheriwse, same as VarintSize32(). + // If negative, 10 bytes. Otherwise, same as VarintSize32(). static size_t VarintSize32SignExtended(int32 value); // Compile-time equivalent of VarintSize32(). @@ -800,87 +1181,73 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { (Value < (1 << 7)) ? 1 : (Value < (1 << 14)) - ? 2 - : (Value < (1 << 21)) - ? 3 - : (Value < (1 << 28)) - ? 4 - : 5; + ? 2 + : (Value < (1 << 21)) ? 3 : (Value < (1 << 28)) ? 4 : 5; }; // Returns the total number of bytes written since this object was created. - inline int ByteCount() const; + int ByteCount() const { + return static_cast(impl_.ByteCount(cur_) - start_count_); + } - // Returns true if there was an underlying I/O error since this object was - // created. - bool HadError() const { return had_error_; } + // Instructs the CodedOutputStream to allow the underlying + // ZeroCopyOutputStream to hold pointers to the original structure instead of + // copying, if it supports it (i.e. output->AllowsAliasing() is true). If the + // underlying stream does not support aliasing, then enabling it has no + // affect. For now, this only affects the behavior of + // WriteRawMaybeAliased(). + // + // NOTE: It is caller's responsibility to ensure that the chunk of memory + // remains live until all of the data has been consumed from the stream. + void EnableAliasing(bool enabled) { impl_.EnableAliasing(enabled); } - // Deterministic serialization, if requested, guarantees that for a given - // binary, equal messages will always be serialized to the same bytes. This - // implies: - // . repeated serialization of a message will return the same bytes - // . different processes of the same binary (which may be executing on - // different machines) will serialize equal messages to the same bytes. + // Indicate to the serializer whether the user wants derministic + // serialization. The default when this is not called comes from the global + // default, controlled by SetDefaultSerializationDeterministic. // - // Note the deterministic serialization is NOT canonical across languages; it - // is also unstable across different builds with schema changes due to unknown - // fields. Users who need canonical serialization, e.g., persistent storage in - // a canonical form, fingerprinting, etc., should define their own - // canonicalization specification and implement the serializer using - // reflection APIs rather than relying on this API. + // What deterministic serialization means is entirely up to the driver of the + // serialization process (i.e. the caller of methods like WriteVarint32). In + // the case of serializing a proto buffer message using one of the methods of + // MessageLite, this means that for a given binary equal messages will always + // be serialized to the same bytes. This implies: + // + // * Repeated serialization of a message will return the same bytes. // - // If deterministic serialization is requested, the serializer will - // sort map entries by keys in lexicographical order or numerical order. - // (This is an implementation detail and may subject to change.) + // * Different processes running the same binary (including on different + // machines) will serialize equal messages to the same bytes. // - // There are two ways to determine whether serialization should be - // deterministic for this CodedOutputStream. If SetSerializationDeterministic - // has not yet been called, then the default comes from the global default, - // which is false, until SetDefaultSerializationDeterministic has been called. - // Otherwise, SetSerializationDeterministic has been called, and the last - // value passed to it is all that matters. + // Note that this is *not* canonical across languages. It is also unstable + // across different builds with intervening message definition changes, due to + // unknown fields. Users who need canonical serialization (e.g. persistent + // storage in a canonical form, fingerprinting) should define their own + // canonicalization specification and implement the serializer using + // reflection APIs rather than relying on this API. void SetSerializationDeterministic(bool value) { - is_serialization_deterministic_ = value; + impl_.SetSerializationDeterministic(value); } - // See above. Also, note that users of this CodedOutputStream may need to - // call IsSerializationDeterministic() to serialize in the intended way. This - // CodedOutputStream cannot enforce a desire for deterministic serialization - // by itself. + + // Return whether the user wants deterministic serialization. See above. bool IsSerializationDeterministic() const { - return is_serialization_deterministic_; + return impl_.IsSerializationDeterministic(); } static bool IsDefaultSerializationDeterministic() { - return default_serialization_deterministic_.load(std::memory_order_relaxed) != 0; + return default_serialization_deterministic_.load( + std::memory_order_relaxed) != 0; } - private: - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream); - - ZeroCopyOutputStream* output_; - uint8* buffer_; - int buffer_size_; - int total_bytes_; // Sum of sizes of all buffers seen so far. - bool had_error_; // Whether an error occurred during output. - bool aliasing_enabled_; // See EnableAliasing(). - bool is_serialization_deterministic_; - static std::atomic default_serialization_deterministic_; - - // Advance the buffer by a given number of bytes. - void Advance(int amount); - - // Called when the buffer runs out to request more data. Implies an - // Advance(buffer_size_). - bool Refresh(); + template + void Serialize(const Func& func); - // Like WriteRaw() but may avoid copying if the underlying - // ZeroCopyOutputStream supports it. - void WriteAliasedRaw(const void* buffer, int size); + uint8* Cur() const { return cur_; } + void SetCur(uint8* ptr) { cur_ = ptr; } + EpsCopyOutputStream* EpsCopy() { return &impl_; } - // If this write might cross the end of the buffer, we compose the bytes first - // then use WriteRaw(). - void WriteVarint32SlowPath(uint32 value); - void WriteVarint64SlowPath(uint64 value); + private: + EpsCopyOutputStream impl_; + uint8* cur_; + int64 start_count_; + static std::atomic default_serialization_deterministic_; // See above. Other projects may use "friend" to allow them to call this. // After SetDefaultSerializationDeterministic() completes, all protocol @@ -889,10 +1256,11 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // that wants deterministic serialization by default needs to call // SetDefaultSerializationDeterministic() or ensure on its own that another // thread has done so. - friend void ::google::protobuf::internal::MapTestForceDeterministic(); + friend void internal::MapTestForceDeterministic(); static void SetDefaultSerializationDeterministic() { default_serialization_deterministic_.store(true, std::memory_order_relaxed); } + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream); }; // inline methods ==================================================== @@ -901,7 +1269,7 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { inline bool CodedInputStream::ReadVarint32(uint32* value) { uint32 v = 0; - if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) { + if (PROTOBUF_PREDICT_TRUE(buffer_ < buffer_end_)) { v = *buffer_; if (v < 0x80) { *value = v; @@ -915,7 +1283,7 @@ inline bool CodedInputStream::ReadVarint32(uint32* value) { } inline bool CodedInputStream::ReadVarint64(uint64* value) { - if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_) && *buffer_ < 0x80) { + if (PROTOBUF_PREDICT_TRUE(buffer_ < buffer_end_) && *buffer_ < 0x80) { *value = *buffer_; Advance(1); return true; @@ -926,7 +1294,7 @@ inline bool CodedInputStream::ReadVarint64(uint64* value) { } inline bool CodedInputStream::ReadVarintSizeAsInt(int* value) { - if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) { + if (PROTOBUF_PREDICT_TRUE(buffer_ < buffer_end_)) { int v = *buffer_; if (v < 0x80) { *value = v; @@ -940,14 +1308,13 @@ inline bool CodedInputStream::ReadVarintSizeAsInt(int* value) { // static inline const uint8* CodedInputStream::ReadLittleEndian32FromArray( - const uint8* buffer, - uint32* value) { + const uint8* buffer, uint32* value) { #if defined(PROTOBUF_LITTLE_ENDIAN) memcpy(value, buffer, sizeof(*value)); return buffer + sizeof(*value); #else - *value = (static_cast(buffer[0]) ) | - (static_cast(buffer[1]) << 8) | + *value = (static_cast(buffer[0])) | + (static_cast(buffer[1]) << 8) | (static_cast(buffer[2]) << 16) | (static_cast(buffer[3]) << 24); return buffer + sizeof(*value); @@ -955,29 +1322,27 @@ inline const uint8* CodedInputStream::ReadLittleEndian32FromArray( } // static inline const uint8* CodedInputStream::ReadLittleEndian64FromArray( - const uint8* buffer, - uint64* value) { + const uint8* buffer, uint64* value) { #if defined(PROTOBUF_LITTLE_ENDIAN) memcpy(value, buffer, sizeof(*value)); return buffer + sizeof(*value); #else - uint32 part0 = (static_cast(buffer[0]) ) | - (static_cast(buffer[1]) << 8) | + uint32 part0 = (static_cast(buffer[0])) | + (static_cast(buffer[1]) << 8) | (static_cast(buffer[2]) << 16) | (static_cast(buffer[3]) << 24); - uint32 part1 = (static_cast(buffer[4]) ) | - (static_cast(buffer[5]) << 8) | + uint32 part1 = (static_cast(buffer[4])) | + (static_cast(buffer[5]) << 8) | (static_cast(buffer[6]) << 16) | (static_cast(buffer[7]) << 24); - *value = static_cast(part0) | - (static_cast(part1) << 32); + *value = static_cast(part0) | (static_cast(part1) << 32); return buffer + sizeof(*value); #endif } inline bool CodedInputStream::ReadLittleEndian32(uint32* value) { #if defined(PROTOBUF_LITTLE_ENDIAN) - if (GOOGLE_PREDICT_TRUE(BufferSize() >= static_cast(sizeof(*value)))) { + if (PROTOBUF_PREDICT_TRUE(BufferSize() >= static_cast(sizeof(*value)))) { buffer_ = ReadLittleEndian32FromArray(buffer_, value); return true; } else { @@ -990,7 +1355,7 @@ inline bool CodedInputStream::ReadLittleEndian32(uint32* value) { inline bool CodedInputStream::ReadLittleEndian64(uint64* value) { #if defined(PROTOBUF_LITTLE_ENDIAN) - if (GOOGLE_PREDICT_TRUE(BufferSize() >= static_cast(sizeof(*value)))) { + if (PROTOBUF_PREDICT_TRUE(BufferSize() >= static_cast(sizeof(*value)))) { buffer_ = ReadLittleEndian64FromArray(buffer_, value); return true; } else { @@ -1003,7 +1368,7 @@ inline bool CodedInputStream::ReadLittleEndian64(uint64* value) { inline uint32 CodedInputStream::ReadTagNoLastTag() { uint32 v = 0; - if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) { + if (PROTOBUF_PREDICT_TRUE(buffer_ < buffer_end_)) { v = *buffer_; if (v < 0x80) { Advance(1); @@ -1020,7 +1385,7 @@ inline std::pair CodedInputStream::ReadTagWithCutoffNoLastTag( // constant, and things like "cutoff >= kMax1ByteVarint" to be evaluated at // compile time. uint32 first_byte_or_zero = 0; - if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) { + if (PROTOBUF_PREDICT_TRUE(buffer_ < buffer_end_)) { // Hot case: buffer_ non_empty, buffer_[0] in [1, 128). // TODO(gpike): Is it worth rearranging this? E.g., if the number of fields // is large enough then is it better to check for the two-byte case first? @@ -1034,8 +1399,8 @@ inline std::pair CodedInputStream::ReadTagWithCutoffNoLastTag( // Other hot case: cutoff >= 0x80, buffer_ has at least two bytes available, // and tag is two bytes. The latter is tested by bitwise-and-not of the // first byte and the second byte. - if (cutoff >= 0x80 && GOOGLE_PREDICT_TRUE(buffer_ + 1 < buffer_end_) && - GOOGLE_PREDICT_TRUE((buffer_[0] & ~buffer_[1]) >= 0x80)) { + if (cutoff >= 0x80 && PROTOBUF_PREDICT_TRUE(buffer_ + 1 < buffer_end_) && + PROTOBUF_PREDICT_TRUE((buffer_[0] & ~buffer_[1]) >= 0x80)) { const uint32 kMax2ByteVarint = (0x7f << 7) + 0x7f; uint32 tag = (1u << 7) * buffer_[1] + (buffer_[0] - 0x80); Advance(2); @@ -1064,14 +1429,15 @@ inline bool CodedInputStream::ConsumedEntireMessage() { inline bool CodedInputStream::ExpectTag(uint32 expected) { if (expected < (1 << 7)) { - if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_) && buffer_[0] == expected) { + if (PROTOBUF_PREDICT_TRUE(buffer_ < buffer_end_) && + buffer_[0] == expected) { Advance(1); return true; } else { return false; } } else if (expected < (1 << 14)) { - if (GOOGLE_PREDICT_TRUE(BufferSize() >= 2) && + if (PROTOBUF_PREDICT_TRUE(BufferSize() >= 2) && buffer_[0] == static_cast(expected | 0x80) && buffer_[1] == static_cast(expected >> 7)) { Advance(2); @@ -1085,8 +1451,8 @@ inline bool CodedInputStream::ExpectTag(uint32 expected) { } } -inline const uint8* CodedInputStream::ExpectTagFromArray( - const uint8* buffer, uint32 expected) { +inline const uint8* CodedInputStream::ExpectTagFromArray(const uint8* buffer, + uint32 expected) { if (expected < (1 << 7)) { if (buffer[0] == expected) { return buffer + 1; @@ -1097,7 +1463,7 @@ inline const uint8* CodedInputStream::ExpectTagFromArray( return buffer + 2; } } - return NULL; + return nullptr; } inline void CodedInputStream::GetDirectBufferPointerInline(const void** data, @@ -1110,9 +1476,8 @@ inline bool CodedInputStream::ExpectAtEnd() { // If we are at a limit we know no more bytes can be read. Otherwise, it's // hard to say without calling Refresh(), and we'd rather not do that. - if (buffer_ == buffer_end_ && - ((buffer_size_after_limit_ != 0) || - (total_bytes_read_ == current_limit_))) { + if (buffer_ == buffer_end_ && ((buffer_size_after_limit_ != 0) || + (total_bytes_read_ == current_limit_))) { last_tag_ = 0; // Pretend we called ReadTag()... legitimate_message_end_ = true; // ... and it hit EOF. return true; @@ -1125,36 +1490,109 @@ inline int CodedInputStream::CurrentPosition() const { return total_bytes_read_ - (BufferSize() + buffer_size_after_limit_); } -inline uint8* CodedOutputStream::GetDirectBufferForNBytesAndAdvance(int size) { - if (buffer_size_ < size) { - return NULL; - } else { - uint8* result = buffer_; - Advance(size); - return result; +inline void CodedInputStream::Advance(int amount) { buffer_ += amount; } + +inline void CodedInputStream::SetRecursionLimit(int limit) { + recursion_budget_ += limit - recursion_limit_; + recursion_limit_ = limit; +} + +inline bool CodedInputStream::IncrementRecursionDepth() { + --recursion_budget_; + return recursion_budget_ >= 0; +} + +inline void CodedInputStream::DecrementRecursionDepth() { + if (recursion_budget_ < recursion_limit_) ++recursion_budget_; +} + +inline void CodedInputStream::UnsafeDecrementRecursionDepth() { + assert(recursion_budget_ < recursion_limit_); + ++recursion_budget_; +} + +inline void CodedInputStream::SetExtensionRegistry(const DescriptorPool* pool, + MessageFactory* factory) { + extension_pool_ = pool; + extension_factory_ = factory; +} + +inline const DescriptorPool* CodedInputStream::GetExtensionPool() { + return extension_pool_; +} + +inline MessageFactory* CodedInputStream::GetExtensionFactory() { + return extension_factory_; +} + +inline int CodedInputStream::BufferSize() const { + return static_cast(buffer_end_ - buffer_); +} + +inline CodedInputStream::CodedInputStream(ZeroCopyInputStream* input) + : buffer_(nullptr), + buffer_end_(nullptr), + input_(input), + total_bytes_read_(0), + overflow_bytes_(0), + last_tag_(0), + legitimate_message_end_(false), + aliasing_enabled_(false), + current_limit_(kint32max), + buffer_size_after_limit_(0), + total_bytes_limit_(kDefaultTotalBytesLimit), + recursion_budget_(default_recursion_limit_), + recursion_limit_(default_recursion_limit_), + extension_pool_(nullptr), + extension_factory_(nullptr) { + // Eagerly Refresh() so buffer space is immediately available. + Refresh(); +} + +inline CodedInputStream::CodedInputStream(const uint8* buffer, int size) + : buffer_(buffer), + buffer_end_(buffer + size), + input_(nullptr), + total_bytes_read_(size), + overflow_bytes_(0), + last_tag_(0), + legitimate_message_end_(false), + aliasing_enabled_(false), + current_limit_(size), + buffer_size_after_limit_(0), + total_bytes_limit_(kDefaultTotalBytesLimit), + recursion_budget_(default_recursion_limit_), + recursion_limit_(default_recursion_limit_), + extension_pool_(nullptr), + extension_factory_(nullptr) { + // Note that setting current_limit_ == size is important to prevent some + // code paths from trying to access input_ and segfaulting. +} + +inline bool CodedInputStream::IsFlat() const { return input_ == nullptr; } + +inline bool CodedInputStream::Skip(int count) { + if (count < 0) return false; // security: count is often user-supplied + + const int original_buffer_size = BufferSize(); + + if (count <= original_buffer_size) { + // Just skipping within the current buffer. Easy. + Advance(count); + return true; } + + return SkipFallback(count, original_buffer_size); } inline uint8* CodedOutputStream::WriteVarint32ToArray(uint32 value, uint8* target) { - while (value >= 0x80) { - *target = static_cast(value | 0x80); - value >>= 7; - ++target; - } - *target = static_cast(value); - return target + 1; + return EpsCopyOutputStream::UnsafeVarint(value, target); } inline uint8* CodedOutputStream::WriteVarint64ToArray(uint64 value, uint8* target) { - while (value >= 0x80) { - *target = static_cast(value | 0x80); - value >>= 7; - ++target; - } - *target = static_cast(value); - return target + 1; + return EpsCopyOutputStream::UnsafeVarint(value, target); } inline void CodedOutputStream::WriteVarint32SignExtended(int32 value) { @@ -1172,7 +1610,7 @@ inline uint8* CodedOutputStream::WriteLittleEndian32ToArray(uint32 value, memcpy(target, &value, sizeof(value)); #else target[0] = static_cast(value); - target[1] = static_cast(value >> 8); + target[1] = static_cast(value >> 8); target[2] = static_cast(value >> 16); target[3] = static_cast(value >> 24); #endif @@ -1188,11 +1626,11 @@ inline uint8* CodedOutputStream::WriteLittleEndian64ToArray(uint64 value, uint32 part1 = static_cast(value >> 32); target[0] = static_cast(part0); - target[1] = static_cast(part0 >> 8); + target[1] = static_cast(part0 >> 8); target[2] = static_cast(part0 >> 16); target[3] = static_cast(part0 >> 24); target[4] = static_cast(part1); - target[5] = static_cast(part1 >> 8); + target[5] = static_cast(part1 >> 8); target[6] = static_cast(part1 >> 16); target[7] = static_cast(part1 >> 24); #endif @@ -1200,37 +1638,18 @@ inline uint8* CodedOutputStream::WriteLittleEndian64ToArray(uint64 value, } inline void CodedOutputStream::WriteVarint32(uint32 value) { - if (buffer_size_ >= 5) { - // Fast path: We have enough bytes left in the buffer to guarantee that - // this write won't cross the end, so we can skip the checks. - uint8* target = buffer_; - uint8* end = WriteVarint32ToArray(value, target); - int size = static_cast(end - target); - Advance(size); - } else { - WriteVarint32SlowPath(value); - } + cur_ = impl_.EnsureSpace(cur_); + SetCur(WriteVarint32ToArray(value, Cur())); } inline void CodedOutputStream::WriteVarint64(uint64 value) { - if (buffer_size_ >= 10) { - // Fast path: We have enough bytes left in the buffer to guarantee that - // this write won't cross the end, so we can skip the checks. - uint8* target = buffer_; - uint8* end = WriteVarint64ToArray(value, target); - int size = static_cast(end - target); - Advance(size); - } else { - WriteVarint64SlowPath(value); - } + cur_ = impl_.EnsureSpace(cur_); + SetCur(WriteVarint64ToArray(value, Cur())); } -inline void CodedOutputStream::WriteTag(uint32 value) { - WriteVarint32(value); -} +inline void CodedOutputStream::WriteTag(uint32 value) { WriteVarint32(value); } -inline uint8* CodedOutputStream::WriteTagToArray( - uint32 value, uint8* target) { +inline uint8* CodedOutputStream::WriteTagToArray(uint32 value, uint8* target) { return WriteVarint32ToArray(value, target); } @@ -1256,145 +1675,40 @@ inline size_t CodedOutputStream::VarintSize64(uint64 value) { inline size_t CodedOutputStream::VarintSize32SignExtended(int32 value) { if (value < 0) { - return 10; // TODO(kenton): Make this a symbolic constant. + return 10; // TODO(kenton): Make this a symbolic constant. } else { return VarintSize32(static_cast(value)); } } -inline void CodedOutputStream::WriteString(const string& str) { +inline void CodedOutputStream::WriteString(const std::string& str) { WriteRaw(str.data(), static_cast(str.size())); } -inline void CodedOutputStream::WriteRawMaybeAliased( - const void* data, int size) { - if (aliasing_enabled_) { - WriteAliasedRaw(data, size); - } else { - WriteRaw(data, size); - } -} - -inline uint8* CodedOutputStream::WriteStringToArray( - const string& str, uint8* target) { - return WriteRawToArray(str.data(), static_cast(str.size()), target); -} - -inline int CodedOutputStream::ByteCount() const { - return total_bytes_ - buffer_size_; -} - -inline void CodedInputStream::Advance(int amount) { - buffer_ += amount; -} - -inline void CodedOutputStream::Advance(int amount) { - buffer_ += amount; - buffer_size_ -= amount; -} - -inline void CodedInputStream::SetRecursionLimit(int limit) { - recursion_budget_ += limit - recursion_limit_; - recursion_limit_ = limit; -} - -inline bool CodedInputStream::IncrementRecursionDepth() { - --recursion_budget_; - return recursion_budget_ >= 0; -} - -inline void CodedInputStream::DecrementRecursionDepth() { - if (recursion_budget_ < recursion_limit_) ++recursion_budget_; -} - -inline void CodedInputStream::UnsafeDecrementRecursionDepth() { - assert(recursion_budget_ < recursion_limit_); - ++recursion_budget_; +inline void CodedOutputStream::WriteRawMaybeAliased(const void* data, + int size) { + cur_ = impl_.WriteRawMaybeAliased(data, size, cur_); } -inline void CodedInputStream::SetExtensionRegistry(const DescriptorPool* pool, - MessageFactory* factory) { - extension_pool_ = pool; - extension_factory_ = factory; +inline uint8* CodedOutputStream::WriteRawToArray(const void* data, int size, + uint8* target) { + memcpy(target, data, size); + return target + size; } -inline const DescriptorPool* CodedInputStream::GetExtensionPool() { - return extension_pool_; -} - -inline MessageFactory* CodedInputStream::GetExtensionFactory() { - return extension_factory_; -} - -inline int CodedInputStream::BufferSize() const { - return static_cast(buffer_end_ - buffer_); -} - -inline CodedInputStream::CodedInputStream(ZeroCopyInputStream* input) - : buffer_(NULL), - buffer_end_(NULL), - input_(input), - total_bytes_read_(0), - overflow_bytes_(0), - last_tag_(0), - legitimate_message_end_(false), - aliasing_enabled_(false), - current_limit_(kint32max), - buffer_size_after_limit_(0), - total_bytes_limit_(kDefaultTotalBytesLimit), - recursion_budget_(default_recursion_limit_), - recursion_limit_(default_recursion_limit_), - extension_pool_(NULL), - extension_factory_(NULL) { - // Eagerly Refresh() so buffer space is immediately available. - Refresh(); -} - -inline CodedInputStream::CodedInputStream(const uint8* buffer, int size) - : buffer_(buffer), - buffer_end_(buffer + size), - input_(NULL), - total_bytes_read_(size), - overflow_bytes_(0), - last_tag_(0), - legitimate_message_end_(false), - aliasing_enabled_(false), - current_limit_(size), - buffer_size_after_limit_(0), - total_bytes_limit_(kDefaultTotalBytesLimit), - recursion_budget_(default_recursion_limit_), - recursion_limit_(default_recursion_limit_), - extension_pool_(NULL), - extension_factory_(NULL) { - // Note that setting current_limit_ == size is important to prevent some - // code paths from trying to access input_ and segfaulting. -} - -inline bool CodedInputStream::IsFlat() const { - return input_ == NULL; -} - -inline bool CodedInputStream::Skip(int count) { - if (count < 0) return false; // security: count is often user-supplied - - const int original_buffer_size = BufferSize(); - - if (count <= original_buffer_size) { - // Just skipping within the current buffer. Easy. - Advance(count); - return true; - } - - return SkipFallback(count, original_buffer_size); +inline uint8* CodedOutputStream::WriteStringToArray(const std::string& str, + uint8* target) { + return WriteRawToArray(str.data(), static_cast(str.size()), target); } } // namespace io } // namespace protobuf - +} // namespace google #if defined(_MSC_VER) && _MSC_VER >= 1300 && !defined(__INTEL_COMPILER) - #pragma runtime_checks("c", restore) +#pragma runtime_checks("c", restore) #endif // _MSC_VER && !defined(__INTEL_COMPILER) -} // namespace google +#include + #endif // GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/io_win32.h b/third_party/protobuf-lite/google/protobuf/io/io_win32.h similarity index 58% rename from third_party/protobuf-lite/google/protobuf/stubs/io_win32.h rename to third_party/protobuf-lite/google/protobuf/io/io_win32.h index 9e17d253..7d11dc2b 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/io_win32.h +++ b/third_party/protobuf-lite/google/protobuf/io/io_win32.h @@ -29,68 +29,90 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Author: laszlocsomor@google.com (Laszlo Csomor) -// +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + // This file contains the declarations for Windows implementations of // commonly used POSIX functions such as open(2) and access(2), as well // as macro definitions for flags of these functions. // // By including this file you'll redefine open/access/etc. to -// ::google::protobuf::internal::win32::{open/access/etc.}. +// ::google::protobuf::io::win32::{open/access/etc.}. // Make sure you don't include a header that attempts to redeclare or // redefine these functions, that'll lead to confusing compilation // errors. It's best to #include this file as the last one to ensure that. // // This file is only used on Windows, it's empty on other platforms. -#ifndef GOOGLE_PROTOBUF_STUBS_IO_WIN32_H__ -#define GOOGLE_PROTOBUF_STUBS_IO_WIN32_H__ +#ifndef GOOGLE_PROTOBUF_IO_IO_WIN32_H__ +#define GOOGLE_PROTOBUF_IO_IO_WIN32_H__ #if defined(_WIN32) +#include #include -#include + +#include +#include // Compilers on Windows other than MSVC (e.g. Cygwin, MinGW32) define the // following functions already, except for mkdir. namespace google { namespace protobuf { -namespace internal { +namespace io { namespace win32 { -LIBPROTOBUF_EXPORT FILE* fopen(const char* path, const char* mode); -LIBPROTOBUF_EXPORT int access(const char* path, int mode); -LIBPROTOBUF_EXPORT int chdir(const char* path); -LIBPROTOBUF_EXPORT int close(int fd); -LIBPROTOBUF_EXPORT int dup(int fd); -LIBPROTOBUF_EXPORT int dup2(int fd1, int fd2); -LIBPROTOBUF_EXPORT int mkdir(const char* path, int _mode); -LIBPROTOBUF_EXPORT int open(const char* path, int flags, int mode = 0); -LIBPROTOBUF_EXPORT int read(int fd, void* buffer, size_t size); -LIBPROTOBUF_EXPORT int setmode(int fd, int mode); -LIBPROTOBUF_EXPORT int stat(const char* path, struct _stat* buffer); -LIBPROTOBUF_EXPORT int write(int fd, const void* buffer, size_t size); -LIBPROTOBUF_EXPORT std::wstring testonly_utf8_to_winpath(const char* path); +PROTOBUF_EXPORT FILE* fopen(const char* path, const char* mode); +PROTOBUF_EXPORT int access(const char* path, int mode); +PROTOBUF_EXPORT int chdir(const char* path); +PROTOBUF_EXPORT int close(int fd); +PROTOBUF_EXPORT int dup(int fd); +PROTOBUF_EXPORT int dup2(int fd1, int fd2); +PROTOBUF_EXPORT int mkdir(const char* path, int _mode); +PROTOBUF_EXPORT int open(const char* path, int flags, int mode = 0); +PROTOBUF_EXPORT int read(int fd, void* buffer, size_t size); +PROTOBUF_EXPORT int setmode(int fd, int mode); +PROTOBUF_EXPORT int stat(const char* path, struct _stat* buffer); +PROTOBUF_EXPORT int write(int fd, const void* buffer, size_t size); +PROTOBUF_EXPORT std::wstring testonly_utf8_to_winpath(const char* path); + +enum class ExpandWildcardsResult { + kSuccess = 0, + kErrorNoMatchingFile = 1, + kErrorInputPathConversion = 2, + kErrorOutputPathConversion = 3, +}; + +// Expand wildcards in a path pattern, feed the result to a consumer function. +// +// `path` must be a valid, Windows-style path. It may be absolute, or relative +// to the current working directory, and it may contain wildcards ("*" and "?") +// in the last path segment. This function passes all matching file names to +// `consume`. The resulting paths may not be absolute nor normalized. +// +// The function returns a value from `ExpandWildcardsResult`. +PROTOBUF_EXPORT ExpandWildcardsResult ExpandWildcards( + const std::string& path, std::function consume); namespace strings { // Convert from UTF-16 to Active-Code-Page-encoded or to UTF-8-encoded text. -LIBPROTOBUF_EXPORT bool wcs_to_mbs( - const wchar_t* s, std::string* out, bool outUtf8); +PROTOBUF_EXPORT bool wcs_to_mbs(const wchar_t* s, std::string* out, + bool outUtf8); // Convert from Active-Code-Page-encoded or UTF-8-encoded text to UTF-16. -LIBPROTOBUF_EXPORT bool mbs_to_wcs( - const char* s, std::wstring* out, bool inUtf8); +PROTOBUF_EXPORT bool mbs_to_wcs(const char* s, std::wstring* out, bool inUtf8); // Convert from UTF-8-encoded text to UTF-16. -LIBPROTOBUF_EXPORT bool utf8_to_wcs(const char* input, std::wstring* out); +PROTOBUF_EXPORT bool utf8_to_wcs(const char* input, std::wstring* out); // Convert from UTF-16-encoded text to UTF-8. -LIBPROTOBUF_EXPORT bool wcs_to_utf8(const wchar_t* input, std::string* out); +PROTOBUF_EXPORT bool wcs_to_utf8(const wchar_t* input, std::string* out); } // namespace strings } // namespace win32 -} // namespace internal +} // namespace io } // namespace protobuf } // namespace google @@ -110,6 +132,8 @@ LIBPROTOBUF_EXPORT bool wcs_to_utf8(const wchar_t* input, std::string* out); #define STDOUT_FILENO 1 #endif +#include + #endif // defined(_WIN32) -#endif // GOOGLE_PROTOBUF_STUBS_IO_WIN32_H__ +#endif // GOOGLE_PROTOBUF_IO_IO_WIN32_H__ diff --git a/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h b/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h index 62ace7ae..d3bd6dae 100644 --- a/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h +++ b/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h @@ -107,11 +107,14 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ + #include + #include +#include -namespace google { +namespace google { namespace protobuf { namespace io { @@ -121,7 +124,7 @@ class ZeroCopyOutputStream; // Abstract interface similar to an input stream but designed to minimize // copying. -class LIBPROTOBUF_EXPORT ZeroCopyInputStream { +class PROTOBUF_EXPORT ZeroCopyInputStream { public: ZeroCopyInputStream() {} virtual ~ZeroCopyInputStream() {} @@ -169,7 +172,7 @@ class LIBPROTOBUF_EXPORT ZeroCopyInputStream { virtual bool Skip(int count) = 0; // Returns the total number of bytes read since this object was created. - virtual int64 ByteCount() const = 0; + virtual int64_t ByteCount() const = 0; private: @@ -178,7 +181,7 @@ class LIBPROTOBUF_EXPORT ZeroCopyInputStream { // Abstract interface similar to an output stream but designed to minimize // copying. -class LIBPROTOBUF_EXPORT ZeroCopyOutputStream { +class PROTOBUF_EXPORT ZeroCopyOutputStream { public: ZeroCopyOutputStream() {} virtual ~ZeroCopyOutputStream() {} @@ -224,7 +227,7 @@ class LIBPROTOBUF_EXPORT ZeroCopyOutputStream { virtual void BackUp(int count) = 0; // Returns the total number of bytes written since this object was created. - virtual int64 ByteCount() const = 0; + virtual int64_t ByteCount() const = 0; // Write a given chunk of data to the output. Some output streams may // implement this in a way that avoids copying. Check AllowsAliasing() before @@ -243,6 +246,8 @@ class LIBPROTOBUF_EXPORT ZeroCopyOutputStream { } // namespace io } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ diff --git a/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h b/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h index 29f63bf0..26572cc5 100644 --- a/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h +++ b/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h @@ -44,15 +44,19 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ + +#include #include #include -#include -#include + #include #include +#include #include +#include + namespace google { namespace protobuf { namespace io { @@ -60,7 +64,7 @@ namespace io { // =================================================================== // A ZeroCopyInputStream backed by an in-memory array of bytes. -class LIBPROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream { +class PROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream { public: // Create an InputStream that returns the bytes pointed to by "data". // "data" remains the property of the caller but must remain valid until @@ -70,12 +74,13 @@ class LIBPROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream { // useful for testing; in production you would probably never want to set // it. ArrayInputStream(const void* data, int size, int block_size = -1); + ~ArrayInputStream() override = default; // implements ZeroCopyInputStream ---------------------------------- - bool Next(const void** data, int* size); - void BackUp(int count); - bool Skip(int count); - int64 ByteCount() const; + bool Next(const void** data, int* size) override; + void BackUp(int count) override; + bool Skip(int count) override; + int64_t ByteCount() const override; private: @@ -84,8 +89,8 @@ class LIBPROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream { const int block_size_; // How many bytes to return at a time. int position_; - int last_returned_size_; // How many bytes we returned last time Next() - // was called (used for error checking only). + int last_returned_size_; // How many bytes we returned last time Next() + // was called (used for error checking only). GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayInputStream); }; @@ -93,7 +98,7 @@ class LIBPROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream { // =================================================================== // A ZeroCopyOutputStream backed by an in-memory array of bytes. -class LIBPROTOBUF_EXPORT ArrayOutputStream : public ZeroCopyOutputStream { +class PROTOBUF_EXPORT ArrayOutputStream : public ZeroCopyOutputStream { public: // Create an OutputStream that writes to the bytes pointed to by "data". // "data" remains the property of the caller but must remain valid until @@ -103,20 +108,21 @@ class LIBPROTOBUF_EXPORT ArrayOutputStream : public ZeroCopyOutputStream { // useful for testing; in production you would probably never want to set // it. ArrayOutputStream(void* data, int size, int block_size = -1); + ~ArrayOutputStream() override = default; // implements ZeroCopyOutputStream --------------------------------- - bool Next(void** data, int* size); - void BackUp(int count); - int64 ByteCount() const; + bool Next(void** data, int* size) override; + void BackUp(int count) override; + int64_t ByteCount() const override; private: - uint8* const data_; // The byte array. - const int size_; // Total size of the array. - const int block_size_; // How many bytes to return at a time. + uint8* const data_; // The byte array. + const int size_; // Total size of the array. + const int block_size_; // How many bytes to return at a time. int position_; - int last_returned_size_; // How many bytes we returned last time Next() - // was called (used for error checking only). + int last_returned_size_; // How many bytes we returned last time Next() + // was called (used for error checking only). GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayOutputStream); }; @@ -124,7 +130,7 @@ class LIBPROTOBUF_EXPORT ArrayOutputStream : public ZeroCopyOutputStream { // =================================================================== // A ZeroCopyOutputStream which appends bytes to a string. -class LIBPROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream { +class PROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream { public: // Create a StringOutputStream which appends bytes to the given string. // The string remains property of the caller, but it is mutated in arbitrary @@ -135,20 +141,18 @@ class LIBPROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream { // Hint: If you call target->reserve(n) before creating the stream, // the first call to Next() will return at least n bytes of buffer // space. - explicit StringOutputStream(string* target); + explicit StringOutputStream(std::string* target); + ~StringOutputStream() override = default; // implements ZeroCopyOutputStream --------------------------------- - bool Next(void** data, int* size); - void BackUp(int count); - int64 ByteCount() const; - - protected: - void SetString(string* target); + bool Next(void** data, int* size) override; + void BackUp(int count) override; + int64_t ByteCount() const override; private: static const int kMinimumSize = 16; - string* target_; + std::string* target_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringOutputStream); }; @@ -170,7 +174,7 @@ class LIBPROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream { // CopyingInputStream implementations should avoid buffering if possible. // CopyingInputStreamAdaptor does its own buffering and will read data // in large blocks. -class LIBPROTOBUF_EXPORT CopyingInputStream { +class PROTOBUF_EXPORT CopyingInputStream { public: virtual ~CopyingInputStream() {} @@ -196,7 +200,7 @@ class LIBPROTOBUF_EXPORT CopyingInputStream { // If you want to read from file descriptors or C++ istreams, this is // already implemented for you: use FileInputStream or IstreamInputStream // respectively. -class LIBPROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream { +class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream { public: // Creates a stream that reads from the given CopyingInputStream. // If a block_size is given, it specifies the number of bytes that @@ -205,17 +209,17 @@ class LIBPROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream // copying_stream unless SetOwnsCopyingStream(true) is called. explicit CopyingInputStreamAdaptor(CopyingInputStream* copying_stream, int block_size = -1); - ~CopyingInputStreamAdaptor(); + ~CopyingInputStreamAdaptor() override; // Call SetOwnsCopyingStream(true) to tell the CopyingInputStreamAdaptor to // delete the underlying CopyingInputStream when it is destroyed. void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; } // implements ZeroCopyInputStream ---------------------------------- - bool Next(const void** data, int* size); - void BackUp(int count); - bool Skip(int count); - int64 ByteCount() const; + bool Next(const void** data, int* size) override; + void BackUp(int count) override; + bool Skip(int count) override; + int64_t ByteCount() const override; private: // Insures that buffer_ is not NULL. @@ -227,7 +231,7 @@ class LIBPROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream CopyingInputStream* copying_stream_; bool owns_copying_stream_; - // True if we have seen a permenant error from the underlying stream. + // True if we have seen a permanent error from the underlying stream. bool failed_; // The current position of copying_stream_, relative to the point where @@ -264,7 +268,7 @@ class LIBPROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream // CopyingOutputStream implementations should avoid buffering if possible. // CopyingOutputStreamAdaptor does its own buffering and will write data // in large blocks. -class LIBPROTOBUF_EXPORT CopyingOutputStream { +class PROTOBUF_EXPORT CopyingOutputStream { public: virtual ~CopyingOutputStream() {} @@ -280,7 +284,7 @@ class LIBPROTOBUF_EXPORT CopyingOutputStream { // If you want to write to file descriptors or C++ ostreams, this is // already implemented for you: use FileOutputStream or OstreamOutputStream // respectively. -class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream { +class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream { public: // Creates a stream that writes to the given Unix file descriptor. // If a block_size is given, it specifies the size of the buffers @@ -288,7 +292,7 @@ class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStrea // is used. explicit CopyingOutputStreamAdaptor(CopyingOutputStream* copying_stream, int block_size = -1); - ~CopyingOutputStreamAdaptor(); + ~CopyingOutputStreamAdaptor() override; // Writes all pending data to the underlying stream. Returns false if a // write error occurred on the underlying stream. (The underlying @@ -300,9 +304,9 @@ class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStrea void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; } // implements ZeroCopyOutputStream --------------------------------- - bool Next(void** data, int* size); - void BackUp(int count); - int64 ByteCount() const; + bool Next(void** data, int* size) override; + void BackUp(int count) override; + int64_t ByteCount() const override; private: // Write the current buffer, if it is present. @@ -316,7 +320,7 @@ class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStrea CopyingOutputStream* copying_stream_; bool owns_copying_stream_; - // True if we have seen a permenant error from the underlying stream. + // True if we have seen a permanent error from the underlying stream. bool failed_; // The current position of copying_stream_, relative to the point where @@ -336,6 +340,31 @@ class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStrea GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingOutputStreamAdaptor); }; +// =================================================================== + +// A ZeroCopyInputStream which wraps some other stream and limits it to +// a particular byte count. +class PROTOBUF_EXPORT LimitingInputStream : public ZeroCopyInputStream { + public: + LimitingInputStream(ZeroCopyInputStream* input, int64 limit); + ~LimitingInputStream() override; + + // implements ZeroCopyInputStream ---------------------------------- + bool Next(const void** data, int* size) override; + void BackUp(int count) override; + bool Skip(int count) override; + int64_t ByteCount() const override; + + + private: + ZeroCopyInputStream* input_; + int64 limit_; // Decreases as we go, becomes negative if we overshoot. + int64 prior_bytes_read_; // Bytes read on underlying stream at construction + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LimitingInputStream); +}; + + // =================================================================== // mutable_string_data() and as_string_data() are workarounds to improve @@ -353,31 +382,25 @@ class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStrea // Return a pointer to mutable characters underlying the given string. The // return value is valid until the next time the string is resized. We // trust the caller to treat the return value as an array of length s->size(). -inline char* mutable_string_data(string* s) { -#ifdef LANG_CXX11 +inline char* mutable_string_data(std::string* s) { // This should be simpler & faster than string_as_array() because the latter // is guaranteed to return NULL when *s is empty, so it has to check for that. return &(*s)[0]; -#else - return string_as_array(s); -#endif } // as_string_data(s) is equivalent to // ({ char* p = mutable_string_data(s); make_pair(p, p != NULL); }) // Sometimes it's faster: in some scenarios p cannot be NULL, and then the // code can avoid that check. -inline std::pair as_string_data(string* s) { - char *p = mutable_string_data(s); -#ifdef LANG_CXX11 +inline std::pair as_string_data(std::string* s) { + char* p = mutable_string_data(s); return std::make_pair(p, true); -#else - return std::make_pair(p, p != NULL); -#endif } } // namespace io } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/map.h b/third_party/protobuf-lite/google/protobuf/map.h index 6463ac2e..8b656ab7 100644 --- a/third_party/protobuf-lite/google/protobuf/map.h +++ b/third_party/protobuf-lite/google/protobuf/map.h @@ -41,6 +41,7 @@ #include #include // To support Visual Studio 2008 #include +#include #include #include @@ -49,6 +50,12 @@ #include #include +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + +#include + namespace google { namespace protobuf { @@ -57,7 +64,8 @@ class Map; class MapIterator; -template struct is_proto_enum; +template +struct is_proto_enum; namespace internal { template +class MapAllocator { + public: + using value_type = U; + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using size_type = size_t; + using difference_type = ptrdiff_t; + + MapAllocator() : arena_(nullptr) {} + explicit MapAllocator(Arena* arena) : arena_(arena) {} + template + MapAllocator(const MapAllocator& allocator) // NOLINT(runtime/explicit) + : arena_(allocator.arena()) {} + + pointer allocate(size_type n, const void* /* hint */ = nullptr) { + // If arena is not given, malloc needs to be called which doesn't + // construct element object. + if (arena_ == nullptr) { + return static_cast(::operator new(n * sizeof(value_type))); + } else { + return reinterpret_cast( + Arena::CreateArray(arena_, n * sizeof(value_type))); + } + } + + void deallocate(pointer p, size_type n) { + if (arena_ == nullptr) { +#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) + ::operator delete(p, n * sizeof(value_type)); +#else + (void)n; + ::operator delete(p); +#endif + } + } + +#if __cplusplus >= 201103L && !defined(GOOGLE_PROTOBUF_OS_APPLE) && \ + !defined(GOOGLE_PROTOBUF_OS_NACL) && \ + !defined(GOOGLE_PROTOBUF_OS_EMSCRIPTEN) + template + void construct(NodeType* p, Args&&... args) { + // Clang 3.6 doesn't compile static casting to void* directly. (Issue + // #1266) According C++ standard 5.2.9/1: "The static_cast operator shall + // not cast away constness". So first the maybe const pointer is casted to + // const void* and after the const void* is const casted. + new (const_cast(static_cast(p))) + NodeType(std::forward(args)...); + } + + template + void destroy(NodeType* p) { + p->~NodeType(); + } +#else + void construct(pointer p, const_reference t) { new (p) value_type(t); } + + void destroy(pointer p) { p->~value_type(); } +#endif + + template + struct rebind { + using other = MapAllocator; + }; + + template + bool operator==(const MapAllocator& other) const { + return arena_ == other.arena_; + } + + template + bool operator!=(const MapAllocator& other) const { + return arena_ != other.arena_; + } + + // To support Visual Studio 2008 + size_type max_size() const { + // parentheses around (std::...:max) prevents macro warning of max() + return (std::numeric_limits::max)(); + } + + // To support gcc-4.4, which does not properly + // support templated friend classes + Arena* arena() const { return arena_; } + + private: + using DestructorSkippable_ = void; + Arena* const arena_; +}; + +template +struct DerefCompare { + bool operator()(const Key* n0, const Key* n1) const { return *n0 < *n1; } +}; + +// This class is used to get trivially destructible views of std::string and +// MapKey, which are the only non-trivially destructible allowed key types. +template +class KeyView { + public: + KeyView(const Key& key) : key_(&key) {} // NOLINT(runtime/explicit) + + const Key& get() const { return *key_; } + // Allows implicit conversions to `const Key&`, which allows us to use the + // hasher defined for Key. + operator const Key&() const { return get(); } // NOLINT(runtime/explicit) + + bool operator==(const KeyView& other) const { return get() == other.get(); } + bool operator==(const Key& other) const { return get() == other; } + bool operator<(const KeyView& other) const { return get() < other.get(); } + bool operator<(const Key& other) const { return get() < other; } + + private: + const Key* key_; +}; + +// Allows the InnerMap type to support skippable destruction. +template +struct GetTrivialKey { + using type = + typename std::conditional::value, Key, + KeyView>::type; +}; + } // namespace internal -// This is the class for google::protobuf::Map's internal value_type. Instead of using +// This is the class for Map's internal value_type. Instead of using // std::pair as value_type, we use this class which provides us more control of // its process of construction and destruction. template -class MapPair { - public: - typedef const Key first_type; - typedef T second_type; +struct MapPair { + using first_type = const Key; + using second_type = T; MapPair(const Key& other_first, const T& other_second) : first(other_first), second(other_second) {} explicit MapPair(const Key& other_first) : first(other_first), second() {} - MapPair(const MapPair& other) - : first(other.first), second(other.second) {} + MapPair(const MapPair& other) : first(other.first), second(other.second) {} ~MapPair() {} // Implicitly convertible to std::pair of compatible types. template - operator std::pair() const { + operator std::pair() const { // NOLINT(runtime/explicit) return std::pair(first, second); } @@ -105,11 +241,11 @@ class MapPair { T second; private: - friend class ::google::protobuf::Arena; + friend class Arena; friend class Map; }; -// google::protobuf::Map is an associative container type used to store protobuf map +// Map is an associative container type used to store protobuf map // fields. Each Map instance may or may not use a different hash function, a // different iteration order, and so on. E.g., please don't examine // implementation details to decide if the following would work: @@ -122,23 +258,23 @@ class MapPair { template class Map { public: - typedef Key key_type; - typedef T mapped_type; - typedef MapPair value_type; + using key_type = Key; + using mapped_type = T; + using value_type = MapPair; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; - typedef size_t size_type; - typedef hash hasher; + using size_type = size_t; + using hasher = hash; - Map() : arena_(NULL), default_enum_value_(0) { Init(); } + Map() : arena_(nullptr), default_enum_value_(0) { Init(); } explicit Map(Arena* arena) : arena_(arena), default_enum_value_(0) { Init(); } Map(const Map& other) - : arena_(NULL), default_enum_value_(other.default_enum_value_) { + : arena_(nullptr), default_enum_value_(other.default_enum_value_) { Init(); insert(other.begin(), other.end()); } @@ -163,139 +299,44 @@ class Map { template Map(const InputIt& first, const InputIt& last) - : arena_(NULL), default_enum_value_(0) { + : arena_(nullptr), default_enum_value_(0) { Init(); insert(first, last); } ~Map() { clear(); - if (arena_ == NULL) { + if (arena_ == nullptr) { delete elements_; } } private: - void Init() { - elements_ = Arena::Create(arena_, 0u, hasher(), Allocator(arena_)); - } - - // re-implement std::allocator to use arena allocator for memory allocation. - // Used for google::protobuf::Map implementation. Users should not use this class - // directly. - template - class MapAllocator { - public: - typedef U value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - - MapAllocator() : arena_(NULL) {} - explicit MapAllocator(Arena* arena) : arena_(arena) {} - template - MapAllocator(const MapAllocator& allocator) - : arena_(allocator.arena()) {} - - pointer allocate(size_type n, const void* /* hint */ = 0) { - // If arena is not given, malloc needs to be called which doesn't - // construct element object. - if (arena_ == NULL) { - return static_cast(::operator new(n * sizeof(value_type))); - } else { - return reinterpret_cast( - Arena::CreateArray(arena_, n * sizeof(value_type))); - } - } - - void deallocate(pointer p, size_type n) { - if (arena_ == NULL) { -#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) - ::operator delete(p, n * sizeof(value_type)); -#else - (void)n; - ::operator delete(p); -#endif - } - } - -#if __cplusplus >= 201103L && !defined(GOOGLE_PROTOBUF_OS_APPLE) && \ - !defined(GOOGLE_PROTOBUF_OS_NACL) && \ - !defined(GOOGLE_PROTOBUF_OS_EMSCRIPTEN) - template - void construct(NodeType* p, Args&&... args) { - // Clang 3.6 doesn't compile static casting to void* directly. (Issue - // #1266) According C++ standard 5.2.9/1: "The static_cast operator shall - // not cast away constness". So first the maybe const pointer is casted to - // const void* and after the const void* is const casted. - new (const_cast(static_cast(p))) - NodeType(std::forward(args)...); - } - - template - void destroy(NodeType* p) { - p->~NodeType(); - } -#else - void construct(pointer p, const_reference t) { new (p) value_type(t); } - - void destroy(pointer p) { p->~value_type(); } -#endif - - template - struct rebind { - typedef MapAllocator other; - }; - - template - bool operator==(const MapAllocator& other) const { - return arena_ == other.arena_; - } - - template - bool operator!=(const MapAllocator& other) const { - return arena_ != other.arena_; - } - - // To support Visual Studio 2008 - size_type max_size() const { - // parentheses around (std::...:max) prevents macro warning of max() - return (std::numeric_limits::max)(); - } - - // To support gcc-4.4, which does not properly - // support templated friend classes - Arena* arena() const { - return arena_; - } - - private: - typedef void DestructorSkippable_; - Arena* const arena_; - }; - - // InnerMap's key type is Key and its value type is value_type*. We use a - // custom class here and for Node, below, to ensure that k_ is at offset 0, - // allowing safe conversion from pointer to Node to pointer to Key, and vice - // versa when appropriate. + void Init() { elements_ = Arena::CreateMessage(arena_, 0u); } + + // InnerMap's key type is TrivialKey and its value type is value_type*. We + // use a custom class here and for Node, below, to ensure that k_ is at offset + // 0, allowing safe conversion from pointer to Node to pointer to TrivialKey, + // and vice versa when appropriate. We use GetTrivialKey to adapt Key to + // be a trivially destructible view if Key is not already trivially + // destructible. This view points into the Key inside v_ once it's + // initialized. + using TrivialKey = typename internal::GetTrivialKey::type; class KeyValuePair { public: - KeyValuePair(const Key& k, value_type* v) : k_(k), v_(v) {} + KeyValuePair(const TrivialKey& k, value_type* v) : k_(k), v_(v) {} - const Key& key() const { return k_; } - Key& key() { return k_; } + const TrivialKey& key() const { return k_; } + TrivialKey& key() { return k_; } value_type* value() const { return v_; } value_type*& value() { return v_; } private: - Key k_; + TrivialKey k_; value_type* v_; }; - typedef MapAllocator Allocator; + using Allocator = internal::MapAllocator; // InnerMap is a generic hash-based map. It doesn't contain any // protocol-buffer-specific logic. It is a chaining hash map with the @@ -312,7 +353,7 @@ class Map { // 2. The number of buckets is a power of two. // 3. Buckets are converted to trees in pairs: if we convert bucket b then // buckets b and b^1 will share a tree. Invariant: buckets b and b^1 have - // the same non-NULL value iff they are sharing a tree. (An alternative + // the same non-null value iff they are sharing a tree. (An alternative // implementation strategy would be to have a tag bit per bucket.) // 4. As is typical for hash_map and such, the Keys and Values are always // stored in linked list nodes. Pointers to elements are never invalidated @@ -322,27 +363,36 @@ class Map { // 6. Once we've tree-converted a bucket, it is never converted back. However, // the items a tree contains may wind up assigned to trees or lists upon a // rehash. - // 7. The code requires no C++ features from C++11 or later. + // 7. The code requires no C++ features from C++14 or later. // 8. Mutations to a map do not invalidate the map's iterators, pointers to // elements, or references to elements. // 9. Except for erase(iterator), any non-const method can reorder iterators. + // 10. InnerMap's key is TrivialKey, which is either Key, if Key is trivially + // destructible, or a trivially destructible view of Key otherwise. This + // allows InnerMap's destructor to be skipped when InnerMap is + // arena-allocated. class InnerMap : private hasher { public: - typedef value_type* Value; + using Value = value_type*; - InnerMap(size_type n, hasher h, Allocator alloc) - : hasher(h), + explicit InnerMap(size_type n) : InnerMap(nullptr, n) {} + InnerMap(Arena* arena, size_type n) + : hasher(), num_elements_(0), seed_(Seed()), - table_(NULL), - alloc_(alloc) { + table_(nullptr), + alloc_(arena) { n = TableSize(n); table_ = CreateEmptyTable(n); num_buckets_ = index_of_first_non_null_ = n; + static_assert( + std::is_trivially_destructible::value, + "We require KeyValuePair to be trivially destructible so that we can " + "skip InnerMap's destructor when it's arena allocated."); } ~InnerMap() { - if (table_ != NULL) { + if (table_ != nullptr) { clear(); Dealloc(table_, num_buckets_); } @@ -359,36 +409,36 @@ class Map { // This is safe only if the given pointer is known to point to a Key that is // part of a Node. - static Node* NodePtrFromKeyPtr(Key* k) { + static Node* NodePtrFromKeyPtr(TrivialKey* k) { return reinterpret_cast(k); } - static Key* KeyPtrFromNodePtr(Node* node) { return &node->kv.key(); } + static TrivialKey* KeyPtrFromNodePtr(Node* node) { return &node->kv.key(); } // Trees. The payload type is pointer to Key, so that we can query the tree // with Keys that are not in any particular data structure. When we insert, // though, the pointer is always pointing to a Key that is inside a Node. - struct KeyCompare { - bool operator()(const Key* n0, const Key* n1) const { return *n0 < *n1; } - }; - typedef typename Allocator::template rebind::other KeyPtrAllocator; - typedef std::set Tree; - typedef typename Tree::iterator TreeIterator; + using KeyPtrAllocator = + typename Allocator::template rebind::other; + using Tree = std::set, + KeyPtrAllocator>; + using TreeIterator = typename Tree::iterator; // iterator and const_iterator are instantiations of iterator_base. template - struct iterator_base { - typedef KeyValueType& reference; - typedef KeyValueType* pointer; + class iterator_base { + public: + using reference = KeyValueType&; + using pointer = KeyValueType*; // Invariants: // node_ is always correct. This is handy because the most common // operations are operator* and operator-> and they only use node_. - // When node_ is set to a non-NULL value, all the other non-const fields + // When node_ is set to a non-null value, all the other non-const fields // are updated to be correct also, but those fields can become stale // if the underlying map is modified. When those fields are needed they // are rechecked, and updated if necessary. - iterator_base() : node_(NULL), m_(NULL), bucket_index_(0) {} + iterator_base() : node_(nullptr), m_(nullptr), bucket_index_(0) {} explicit iterator_base(const InnerMap* m) : m_(m) { SearchFrom(m->index_of_first_non_null_); @@ -408,15 +458,15 @@ class Map { : node_(NodePtrFromKeyPtr(*tree_it)), m_(m), bucket_index_(index) { // Invariant: iterators that use buckets with trees have an even // bucket_index_. - GOOGLE_DCHECK_EQ(bucket_index_ % 2, 0); + GOOGLE_DCHECK_EQ(bucket_index_ % 2, 0u); } // Advance through buckets, looking for the first that isn't empty. - // If nothing non-empty is found then leave node_ == NULL. + // If nothing non-empty is found then leave node_ == nullptr. void SearchFrom(size_type start_bucket) { GOOGLE_DCHECK(m_->index_of_first_non_null_ == m_->num_buckets_ || - m_->table_[m_->index_of_first_non_null_] != NULL); - node_ = NULL; + m_->table_[m_->index_of_first_non_null_] != nullptr); + node_ = nullptr; for (bucket_index_ = start_bucket; bucket_index_ < m_->num_buckets_; bucket_index_++) { if (m_->TableEntryIsNonEmptyList(bucket_index_)) { @@ -442,13 +492,13 @@ class Map { } iterator_base& operator++() { - if (node_->next == NULL) { + if (node_->next == nullptr) { TreeIterator tree_it; const bool is_list = revalidate_if_necessary(&tree_it); if (is_list) { SearchFrom(bucket_index_ + 1); } else { - GOOGLE_DCHECK_EQ(bucket_index_ & 1, 0); + GOOGLE_DCHECK_EQ(bucket_index_ & 1, 0u); Tree* tree = static_cast(m_->table_[bucket_index_]); if (++tree_it == tree->end()) { SearchFrom(bucket_index_ + 2); @@ -468,22 +518,21 @@ class Map { return tmp; } - // Assumes node_ and m_ are correct and non-NULL, but other fields may be + // Assumes node_ and m_ are correct and non-null, but other fields may be // stale. Fix them as needed. Then return true iff node_ points to a // Node in a list. If false is returned then *it is modified to be // a valid iterator for node_. bool revalidate_if_necessary(TreeIterator* it) { - GOOGLE_DCHECK(node_ != NULL && m_ != NULL); + GOOGLE_DCHECK(node_ != nullptr && m_ != nullptr); // Force bucket_index_ to be in range. bucket_index_ &= (m_->num_buckets_ - 1); // Common case: the bucket we think is relevant points to node_. - if (m_->table_[bucket_index_] == static_cast(node_)) - return true; + if (m_->table_[bucket_index_] == static_cast(node_)) return true; // Less common: the bucket is a linked list with node_ somewhere in it, // but not at the head. if (m_->TableEntryIsNonEmptyList(bucket_index_)) { Node* l = static_cast(m_->table_[bucket_index_]); - while ((l = l->next) != NULL) { + while ((l = l->next) != nullptr) { if (l == node_) { return true; } @@ -492,7 +541,7 @@ class Map { // Well, bucket_index_ still might be correct, but probably // not. Revalidate just to be sure. This case is rare enough that we // don't worry about potential optimizations, such as having a custom - // find-like method that compares Node* instead of const Key&. + // find-like method that compares Node* instead of TrivialKey. iterator_base i(m_->find(*KeyPtrFromNodePtr(node_), it)); bucket_index_ = i.bucket_index_; return m_->TableEntryIsList(bucket_index_); @@ -504,8 +553,8 @@ class Map { }; public: - typedef iterator_base iterator; - typedef iterator_base const_iterator; + using iterator = iterator_base; + using const_iterator = iterator_base; iterator begin() { return iterator(this); } iterator end() { return iterator(); } @@ -516,16 +565,16 @@ class Map { for (size_type b = 0; b < num_buckets_; b++) { if (TableEntryIsNonEmptyList(b)) { Node* node = static_cast(table_[b]); - table_[b] = NULL; + table_[b] = nullptr; do { Node* next = node->next; DestroyNode(node); node = next; - } while (node != NULL); + } while (node != nullptr); } else if (TableEntryIsTree(b)) { Tree* tree = static_cast(table_[b]); GOOGLE_DCHECK(table_[b] == table_[b + 1] && (b & 1) == 0); - table_[b] = table_[b + 1] = NULL; + table_[b] = table_[b + 1] = nullptr; typename Tree::iterator tree_it = tree->begin(); do { Node* node = NodePtrFromKeyPtr(*tree_it); @@ -551,14 +600,15 @@ class Map { size_type size() const { return num_elements_; } bool empty() const { return size() == 0; } - iterator find(const Key& k) { return iterator(FindHelper(k).first); } - const_iterator find(const Key& k) const { return find(k, NULL); } + iterator find(const TrivialKey& k) { return iterator(FindHelper(k).first); } + const_iterator find(const TrivialKey& k) const { return find(k, nullptr); } + bool contains(const TrivialKey& k) const { return find(k) != end(); } // In traditional C++ style, this performs "insert if not present." std::pair insert(const KeyValuePair& kv) { std::pair p = FindHelper(kv.key()); // Case 1: key was already present. - if (p.first.node_ != NULL) + if (p.first.node_ != nullptr) return std::make_pair(iterator(p.first), false); // Case 2: insert. if (ResizeIfLoadIsOutOfRange(num_elements_ + 1)) { @@ -574,10 +624,10 @@ class Map { // The same, but if an insertion is necessary then the value portion of the // inserted key-value pair is left uninitialized. - std::pair insert(const Key& k) { + std::pair insert(const TrivialKey& k) { std::pair p = FindHelper(k); // Case 1: key was already present. - if (p.first.node_ != NULL) + if (p.first.node_ != nullptr) return std::make_pair(iterator(p.first), false); // Case 2: insert. if (ResizeIfLoadIsOutOfRange(num_elements_ + 1)) { @@ -585,16 +635,19 @@ class Map { } const size_type b = p.second; // bucket number Node* node = Alloc(1); - typedef typename Allocator::template rebind::other KeyAllocator; + using KeyAllocator = + typename Allocator::template rebind::other; KeyAllocator(alloc_).construct(&node->kv.key(), k); iterator result = InsertUnique(b, node); ++num_elements_; return std::make_pair(result, true); } - Value& operator[](const Key& k) { + // Returns iterator so that outer map can update the TrivialKey to point to + // the Key inside value_type in case TrivialKey is a view type. + iterator operator[](const TrivialKey& k) { KeyValuePair kv(k, Value()); - return insert(kv).first->value(); + return insert(kv).first; } void erase(iterator it) { @@ -617,27 +670,27 @@ class Map { // only because we want index_of_first_non_null_ to be correct. b &= ~static_cast(1); DestroyTree(tree); - table_[b] = table_[b + 1] = NULL; + table_[b] = table_[b + 1] = nullptr; } } DestroyNode(item); --num_elements_; - if (GOOGLE_PREDICT_FALSE(b == index_of_first_non_null_)) { + if (PROTOBUF_PREDICT_FALSE(b == index_of_first_non_null_)) { while (index_of_first_non_null_ < num_buckets_ && - table_[index_of_first_non_null_] == NULL) { + table_[index_of_first_non_null_] == nullptr) { ++index_of_first_non_null_; } } } private: - const_iterator find(const Key& k, TreeIterator* it) const { + const_iterator find(const TrivialKey& k, TreeIterator* it) const { return FindHelper(k, it).first; } - std::pair FindHelper(const Key& k) const { - return FindHelper(k, NULL); + std::pair FindHelper(const TrivialKey& k) const { + return FindHelper(k, nullptr); } - std::pair FindHelper(const Key& k, + std::pair FindHelper(const TrivialKey& k, TreeIterator* it) const { size_type b = BucketNumber(k); if (TableEntryIsNonEmptyList(b)) { @@ -648,15 +701,15 @@ class Map { } else { node = node->next; } - } while (node != NULL); + } while (node != nullptr); } else if (TableEntryIsTree(b)) { GOOGLE_DCHECK_EQ(table_[b], table_[b ^ 1]); b &= ~static_cast(1); Tree* tree = static_cast(table_[b]); - Key* key = const_cast(&k); + TrivialKey* key = const_cast(&k); typename Tree::iterator tree_it = tree->find(key); if (tree_it != tree->end()) { - if (it != NULL) *it = tree_it; + if (it != nullptr) *it = tree_it; return std::make_pair(const_iterator(tree_it, this, b), b); } } @@ -669,7 +722,7 @@ class Map { // bucket. num_elements_ is not modified. iterator InsertUnique(size_type b, Node* node) { GOOGLE_DCHECK(index_of_first_non_null_ == num_buckets_ || - table_[index_of_first_non_null_] != NULL); + table_[index_of_first_non_null_] != nullptr); // In practice, the code that led to this point may have already // determined whether we are inserting into an empty list, a short list, // or whatever. But it's probably cheap enough to recompute that here; @@ -679,7 +732,7 @@ class Map { if (TableEntryIsEmpty(b)) { result = InsertUniqueInList(b, node); } else if (TableEntryIsNonEmptyList(b)) { - if (GOOGLE_PREDICT_FALSE(TableEntryIsTooLong(b))) { + if (PROTOBUF_PREDICT_FALSE(TableEntryIsTooLong(b))) { TreeConvert(b); result = InsertUniqueInTree(b, node); GOOGLE_DCHECK_EQ(result.bucket_index_, b & ~static_cast(1)); @@ -711,12 +764,11 @@ class Map { // Tree. iterator InsertUniqueInTree(size_type b, Node* node) { GOOGLE_DCHECK_EQ(table_[b], table_[b ^ 1]); - // Maintain the invariant that node->next is NULL for all Nodes in Trees. - node->next = NULL; - return iterator(static_cast(table_[b]) - ->insert(KeyPtrFromNodePtr(node)) - .first, - this, b & ~static_cast(1)); + // Maintain the invariant that node->next is null for all Nodes in Trees. + node->next = nullptr; + return iterator( + static_cast(table_[b])->insert(KeyPtrFromNodePtr(node)).first, + this, b & ~static_cast(1)); } // Returns whether it did resize. Currently this is only used when @@ -734,13 +786,13 @@ class Map { // We don't care how many elements are in trees. If a lot are, // we may resize even though there are many empty buckets. In // practice, this seems fine. - if (GOOGLE_PREDICT_FALSE(new_size >= hi_cutoff)) { + if (PROTOBUF_PREDICT_FALSE(new_size >= hi_cutoff)) { if (num_buckets_ <= max_size() / 2) { Resize(num_buckets_ * 2); return true; } - } else if (GOOGLE_PREDICT_FALSE(new_size <= lo_cutoff && - num_buckets_ > kMinTableSize)) { + } else if (PROTOBUF_PREDICT_FALSE(new_size <= lo_cutoff && + num_buckets_ > kMinTableSize)) { size_type lg2_of_size_reduction_factor = 1; // It's possible we want to shrink a lot here... size() could even be 0. // So, estimate how much to shrink by making sure we don't shrink so @@ -785,7 +837,7 @@ class Map { Node* next = node->next; InsertUnique(BucketNumber(*KeyPtrFromNodePtr(node)), node); node = next; - } while (node != NULL); + } while (node != nullptr); } void TransferTree(void* const* table, size_type index) { @@ -820,14 +872,14 @@ class Map { return TableEntryIsList(table_, b); } static bool TableEntryIsEmpty(void* const* table, size_type b) { - return table[b] == NULL; + return table[b] == nullptr; } static bool TableEntryIsNonEmptyList(void* const* table, size_type b) { - return table[b] != NULL && table[b] != table[b ^ 1]; + return table[b] != nullptr && table[b] != table[b ^ 1]; } static bool TableEntryIsTree(void* const* table, size_type b) { return !TableEntryIsEmpty(table, b) && - !TableEntryIsNonEmptyList(table, b); + !TableEntryIsNonEmptyList(table, b); } static bool TableEntryIsList(void* const* table, size_type b) { return !TableEntryIsTree(table, b); @@ -841,8 +893,8 @@ class Map { // create a temporary and use the two-arg construct that's known to exist. // It's clunky, but the compiler should be able to generate more-or-less // the same code. - tree_allocator.construct(tree, - Tree(KeyCompare(), KeyPtrAllocator(alloc_))); + tree_allocator.construct( + tree, Tree(typename Tree::key_compare(), KeyPtrAllocator(alloc_))); // Now the tree is ready to use. size_type count = CopyListToTree(b, tree) + CopyListToTree(b ^ 1, tree); GOOGLE_DCHECK_EQ(count, tree->size()); @@ -854,11 +906,11 @@ class Map { size_type CopyListToTree(size_type b, Tree* tree) { size_type count = 0; Node* node = static_cast(table_[b]); - while (node != NULL) { + while (node != nullptr) { tree->insert(KeyPtrFromNodePtr(node)); ++count; Node* next = node->next; - node->next = NULL; + node->next = nullptr; node = next; } return count; @@ -873,20 +925,19 @@ class Map { do { ++count; node = node->next; - } while (node != NULL); + } while (node != nullptr); // Invariant: no linked list ever is more than kMaxLength in length. GOOGLE_DCHECK_LE(count, kMaxLength); return count >= kMaxLength; } - size_type BucketNumber(const Key& k) const { - // We inherit from hasher, so one-arg operator() provides a hash function. - size_type h = (*const_cast(this))(k); + size_type BucketNumber(const TrivialKey& k) const { + size_type h = hash_function()(k); return (h + seed_) & (num_buckets_ - 1); } - bool IsMatch(const Key& k0, const Key& k1) const { - return std::equal_to()(k0, k1); + bool IsMatch(const TrivialKey& k0, const TrivialKey& k1) const { + return k0 == k1; } // Return a power of two no less than max(kMinTableSize, n). @@ -900,14 +951,14 @@ class Map { // Use alloc_ to allocate an array of n objects of type U. template U* Alloc(size_type n) { - typedef typename Allocator::template rebind::other alloc_type; + using alloc_type = typename Allocator::template rebind::other; return alloc_type(alloc_).allocate(n); } // Use alloc_ to deallocate an array of n objects of type U. template void Dealloc(U* t, size_type n) { - typedef typename Allocator::template rebind::other alloc_type; + using alloc_type = typename Allocator::template rebind::other; alloc_type(alloc_).deallocate(t, n); } @@ -933,14 +984,19 @@ class Map { // Return a randomish value. size_type Seed() const { size_type s = static_cast(reinterpret_cast(this)); -#if defined(__x86_64__) && defined(__GNUC__) +#if defined(__x86_64__) && defined(__GNUC__) && \ + !defined(GOOGLE_PROTOBUF_NO_RDTSC) uint32 hi, lo; - asm("rdtsc" : "=a" (lo), "=d" (hi)); + asm("rdtsc" : "=a"(lo), "=d"(hi)); s += ((static_cast(hi) << 32) | lo); #endif return s; } + friend class Arena; + using InternalArenaConstructable_ = void; + using DestructorSkippable_ = void; + size_type num_elements_; size_type num_buckets_; size_type seed_; @@ -953,21 +1009,19 @@ class Map { public: // Iterators class const_iterator { - typedef typename InnerMap::const_iterator InnerIt; + using InnerIt = typename InnerMap::const_iterator; public: - typedef std::forward_iterator_tag iterator_category; - typedef typename Map::value_type value_type; - typedef ptrdiff_t difference_type; - typedef const value_type* pointer; - typedef const value_type& reference; + using iterator_category = std::forward_iterator_tag; + using value_type = typename Map::value_type; + using difference_type = ptrdiff_t; + using pointer = const value_type*; + using reference = const value_type&; const_iterator() {} explicit const_iterator(const InnerIt& it) : it_(it) {} - const_reference operator*() const { - return *it_->value(); - } + const_reference operator*() const { return *it_->value(); } const_pointer operator->() const { return &(operator*()); } const_iterator& operator++() { @@ -988,14 +1042,14 @@ class Map { }; class iterator { - typedef typename InnerMap::iterator InnerIt; + using InnerIt = typename InnerMap::iterator; public: - typedef std::forward_iterator_tag iterator_category; - typedef typename Map::value_type value_type; - typedef ptrdiff_t difference_type; - typedef value_type* pointer; - typedef value_type& reference; + using iterator_category = std::forward_iterator_tag; + using value_type = typename Map::value_type; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; iterator() {} explicit iterator(const InnerIt& it) : it_(it) {} @@ -1010,7 +1064,7 @@ class Map { iterator operator++(int) { return iterator(it_++); } // Allow implicit conversion to const_iterator. - operator const_iterator() const { + operator const_iterator() const { // NOLINT(runtime/explicit) return const_iterator(typename InnerMap::const_iterator(it_)); } @@ -1044,12 +1098,14 @@ class Map { // Element access T& operator[](const key_type& key) { - value_type** value = &(*elements_)[key]; - if (*value == NULL) { + typename InnerMap::iterator it = (*elements_)[key]; + value_type** value = &it->value(); + if (*value == nullptr) { *value = CreateValueTypeInternal(key); - internal::MapValueInitializer::value, - T>::Initialize((*value)->second, - default_enum_value_); + // We need to update the key in case it's a view type. + it->key() = (*value)->first; + internal::MapValueInitializer::value, T>::Initialize( + (*value)->second, default_enum_value_); } return (*value)->second; } @@ -1074,6 +1130,7 @@ class Map { return const_iterator(iterator(elements_->find(key))); } iterator find(const key_type& key) { return iterator(elements_->find(key)); } + bool contains(const Key& key) const { return elements_->contains(key); } std::pair equal_range( const key_type& key) const { const_iterator it = find(key); @@ -1100,6 +1157,8 @@ class Map { elements_->insert(value.first); if (p.second) { p.first->value() = CreateValueTypeInternal(value); + // We need to update the key in case it's a view type. + p.first->key() = p.first->value()->first; } return std::pair(iterator(p.first), p.second); } @@ -1127,9 +1186,12 @@ class Map { } } iterator erase(iterator pos) { - if (arena_ == NULL) delete pos.operator->(); + value_type* value = pos.operator->(); iterator i = pos++; elements_->erase(i.it_); + // Note: we need to delete the value after erasing from the inner map + // because the inner map's key may be a view of the value's key. + if (arena_ == nullptr) delete value; return pos; } void erase(iterator first, iterator last) { @@ -1173,27 +1235,26 @@ class Map { } value_type* CreateValueTypeInternal(const Key& key) { - if (arena_ == NULL) { + if (arena_ == nullptr) { return new value_type(key); } else { value_type* value = reinterpret_cast( Arena::CreateArray(arena_, sizeof(value_type))); - Arena::CreateInArenaStorage(const_cast(&value->first), arena_); + Arena::CreateInArenaStorage(const_cast(&value->first), arena_, key); Arena::CreateInArenaStorage(&value->second, arena_); - const_cast(value->first) = key; return value; } } value_type* CreateValueTypeInternal(const value_type& value) { - if (arena_ == NULL) { + if (arena_ == nullptr) { return new value_type(value); } else { value_type* p = reinterpret_cast( Arena::CreateArray(arena_, sizeof(value_type))); - Arena::CreateInArenaStorage(const_cast(&p->first), arena_); + Arena::CreateInArenaStorage(const_cast(&p->first), arena_, + value.first); Arena::CreateInArenaStorage(&p->second, arena_); - const_cast(p->first) = value.first; p->second = value.second; return p; } @@ -1203,9 +1264,9 @@ class Map { int default_enum_value_; InnerMap* elements_; - friend class ::google::protobuf::Arena; - typedef void InternalArenaConstructable_; - typedef void DestructorSkippable_; + friend class Arena; + using InternalArenaConstructable_ = void; + using DestructorSkippable_ = void; template + #endif // GOOGLE_PROTOBUF_MAP_H__ diff --git a/third_party/protobuf-lite/google/protobuf/map_entry_lite.h b/third_party/protobuf-lite/google/protobuf/map_entry_lite.h index 85a0bed7..7125ba18 100644 --- a/third_party/protobuf-lite/google/protobuf/map_entry_lite.h +++ b/third_party/protobuf-lite/google/protobuf/map_entry_lite.h @@ -32,14 +32,23 @@ #define GOOGLE_PROTOBUF_MAP_ENTRY_LITE_H__ #include +#include #include +#include +#include #include #include +#include #include #include -#include -#include +#include +#include + +#include +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { @@ -54,7 +63,9 @@ template { // strings and similar } }; +// Functions for operating on a map entry. Does not contain any representation +// (this class is not intended to be instantiated). +template +struct MapEntryFuncs { + typedef MapTypeHandler KeyTypeHandler; + typedef MapTypeHandler ValueTypeHandler; + static const int kKeyFieldNumber = 1; + static const int kValueFieldNumber = 2; + + static uint8* InternalSerialize(int field_number, const Key& key, + const Value& value, uint8* ptr, + io::EpsCopyOutputStream* stream) { + ptr = stream->EnsureSpace(ptr); + ptr = WireFormatLite::WriteTagToArray( + field_number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, ptr); + ptr = io::CodedOutputStream::WriteVarint32ToArray(GetCachedSize(key, value), + ptr); + + ptr = KeyTypeHandler::Write(kKeyFieldNumber, key, ptr, stream); + return ValueTypeHandler::Write(kValueFieldNumber, value, ptr, stream); + } + + static size_t ByteSizeLong(const Key& key, const Value& value) { + // Tags for key and value will both be one byte (field numbers 1 and 2). + size_t inner_length = + 2 + KeyTypeHandler::ByteSize(key) + ValueTypeHandler::ByteSize(value); + return inner_length + io::CodedOutputStream::VarintSize32( + static_cast(inner_length)); + } + + static int GetCachedSize(const Key& key, const Value& value) { + // Tags for key and value will both be one byte (field numbers 1 and 2). + return 2 + KeyTypeHandler::GetCachedSize(key) + + ValueTypeHandler::GetCachedSize(value); + } +}; + // MapEntryImpl is used to implement parsing and serialization of map entries. // It uses Curious Recursive Template Pattern (CRTP) to provide the type of // the eventual code to the template code. @@ -97,6 +146,9 @@ template class MapEntryImpl : public Base { + public: + typedef MapEntryFuncs Funcs; + protected: // Provide utilities to parse/serialize key/value. Provide utilities to // manipulate internal stored type. @@ -111,16 +163,16 @@ class MapEntryImpl : public Base { // Enum type cannot be used for MapTypeHandler::Read. Define a type // which will replace Enum with int. typedef typename KeyTypeHandler::MapEntryAccessorType KeyMapEntryAccessorType; - typedef typename ValueTypeHandler::MapEntryAccessorType - ValueMapEntryAccessorType; + typedef + typename ValueTypeHandler::MapEntryAccessorType ValueMapEntryAccessorType; // Constants for field number. static const int kKeyFieldNumber = 1; static const int kValueFieldNumber = 2; // Constants for field tag. - static const uint8 kKeyTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG( - kKeyFieldNumber, KeyTypeHandler::kWireType); + static const uint8 kKeyTag = + GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(kKeyFieldNumber, KeyTypeHandler::kWireType); static const uint8 kValueTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG( kValueFieldNumber, ValueTypeHandler::kWireType); static const size_t kTagSize = 1; @@ -135,14 +187,14 @@ class MapEntryImpl : public Base { static const WireFormatLite::FieldType kEntryValueFieldType = kValueFieldType; static const int kEntryDefaultEnumValue = default_enum_value; - MapEntryImpl() : arena_(NULL) { + MapEntryImpl() { KeyTypeHandler::Initialize(&key_, NULL); ValueTypeHandler::InitializeMaybeByDefaultEnum(&value_, default_enum_value, NULL); _has_bits_[0] = 0; } - explicit MapEntryImpl(Arena* arena) : arena_(arena) { + explicit MapEntryImpl(Arena* arena) : Base(arena) { KeyTypeHandler::Initialize(&key_, arena); ValueTypeHandler::InitializeMaybeByDefaultEnum(&value_, default_enum_value, arena); @@ -150,7 +202,7 @@ class MapEntryImpl : public Base { } ~MapEntryImpl() { - if (GetArenaNoVirtual() != NULL) return; + if (Base::GetArena() != NULL) return; KeyTypeHandler::DeleteNoArena(key_); ValueTypeHandler::DeleteNoArena(value_); } @@ -166,142 +218,120 @@ class MapEntryImpl : public Base { } inline KeyMapEntryAccessorType* mutable_key() { set_has_key(); - return KeyTypeHandler::EnsureMutable(&key_, GetArenaNoVirtual()); + return KeyTypeHandler::EnsureMutable(&key_, Base::GetArena()); } inline ValueMapEntryAccessorType* mutable_value() { set_has_value(); - return ValueTypeHandler::EnsureMutable(&value_, GetArenaNoVirtual()); + return ValueTypeHandler::EnsureMutable(&value_, Base::GetArena()); } // implements MessageLite ========================================= // MapEntryImpl is for implementation only and this function isn't called // anywhere. Just provide a fake implementation here for MessageLite. - string GetTypeName() const { return ""; } + std::string GetTypeName() const override { return ""; } - void CheckTypeAndMergeFrom(const MessageLite& other) { - MergeFromInternal(*::google::protobuf::down_cast(&other)); + void CheckTypeAndMergeFrom(const MessageLite& other) override { + MergeFromInternal(*::google::protobuf::internal::DownCast(&other)); } - bool MergePartialFromCodedStream(::google::protobuf::io::CodedInputStream* input) { - uint32 tag; - - for (;;) { - // 1) corrupted data: return false; - // 2) unknown field: skip without putting into unknown field set; - // 3) unknown enum value: keep it in parsing. In proto2, caller should - // check the value and put this entry into containing message's unknown - // field set if the value is an unknown enum. In proto3, caller doesn't - // need to care whether the value is unknown enum; - // 4) missing key/value: missed key/value will have default value. caller - // should take this entry as if key/value is set to default value. - tag = input->ReadTagNoLastTag(); - switch (tag) { - case kKeyTag: - if (!KeyTypeHandler::Read(input, mutable_key())) { - return false; - } - set_has_key(); - break; - - case kValueTag: - if (!ValueTypeHandler::Read(input, mutable_value())) { - return false; - } - set_has_value(); - if (input->ExpectAtEnd()) return true; - break; - - default: - if (tag == 0 || - WireFormatLite::GetTagWireType(tag) == - WireFormatLite::WIRETYPE_END_GROUP) { - return true; - } - if (!WireFormatLite::SkipField(input, tag)) return false; - break; + const char* _InternalParse(const char* ptr, ParseContext* ctx) final { + while (!ctx->Done(&ptr)) { + uint32 tag; + ptr = ReadTag(ptr, &tag); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + if (tag == kKeyTag) { + set_has_key(); + KeyMapEntryAccessorType* key = mutable_key(); + ptr = KeyTypeHandler::Read(ptr, ctx, key); + if (!Derived::ValidateKey(key)) return nullptr; + } else if (tag == kValueTag) { + set_has_value(); + ValueMapEntryAccessorType* value = mutable_value(); + ptr = ValueTypeHandler::Read(ptr, ctx, value); + if (!Derived::ValidateValue(value)) return nullptr; + } else { + if (tag == 0 || WireFormatLite::GetTagWireType(tag) == + WireFormatLite::WIRETYPE_END_GROUP) { + ctx->SetLastTag(tag); + return ptr; + } + ptr = UnknownFieldParse(tag, static_cast(nullptr), ptr, + ctx); } + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); } + return ptr; } - size_t ByteSizeLong() const { + size_t ByteSizeLong() const override { size_t size = 0; - size += has_key() ? - kTagSize + static_cast(KeyTypeHandler::ByteSize(key())) : 0; - size += has_value() ? - kTagSize + static_cast(ValueTypeHandler::ByteSize(value())) : 0; + size += has_key() ? kTagSize + + static_cast(KeyTypeHandler::ByteSize(key())) + : 0; + size += has_value() + ? kTagSize + + static_cast(ValueTypeHandler::ByteSize(value())) + : 0; return size; } - void SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream* output) const { - KeyTypeHandler::Write(kKeyFieldNumber, key(), output); - ValueTypeHandler::Write(kValueFieldNumber, value(), output); - } - - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(bool deterministic, - ::google::protobuf::uint8* output) const { - output = KeyTypeHandler::InternalWriteToArray(kKeyFieldNumber, key(), - deterministic, output); - output = ValueTypeHandler::InternalWriteToArray(kValueFieldNumber, value(), - deterministic, output); - return output; + ::google::protobuf::uint8* _InternalSerialize(::google::protobuf::uint8* ptr, + io::EpsCopyOutputStream* stream) const override { + ptr = KeyTypeHandler::Write(kKeyFieldNumber, key(), ptr, stream); + return ValueTypeHandler::Write(kValueFieldNumber, value(), ptr, stream); } // Don't override SerializeWithCachedSizesToArray. Use MessageLite's. - int GetCachedSize() const { + int GetCachedSize() const override { int size = 0; - size += has_key() - ? static_cast(kTagSize) + KeyTypeHandler::GetCachedSize(key()) - : 0; - size += has_value() - ? static_cast(kTagSize) + ValueTypeHandler::GetCachedSize(value()) - : 0; + size += has_key() ? static_cast(kTagSize) + + KeyTypeHandler::GetCachedSize(key()) + : 0; + size += has_value() ? static_cast(kTagSize) + + ValueTypeHandler::GetCachedSize(value()) + : 0; return size; } - bool IsInitialized() const { return ValueTypeHandler::IsInitialized(value_); } + bool IsInitialized() const override { + return ValueTypeHandler::IsInitialized(value_); + } - Base* New() const { + Base* New() const override { Derived* entry = new Derived; return entry; } - Base* New(Arena* arena) const { + Base* New(Arena* arena) const override { Derived* entry = Arena::CreateMessage(arena); return entry; } - size_t SpaceUsedLong() const { - size_t size = sizeof(Derived); - size += KeyTypeHandler::SpaceUsedInMapEntryLong(key_); - size += ValueTypeHandler::SpaceUsedInMapEntryLong(value_); - return size; - } - protected: // We can't declare this function directly here as it would hide the other // overload (const Message&). void MergeFromInternal(const MapEntryImpl& from) { if (from._has_bits_[0]) { if (from.has_key()) { - KeyTypeHandler::EnsureMutable(&key_, GetArenaNoVirtual()); - KeyTypeHandler::Merge(from.key(), &key_, GetArenaNoVirtual()); + KeyTypeHandler::EnsureMutable(&key_, Base::GetArena()); + KeyTypeHandler::Merge(from.key(), &key_, Base::GetArena()); set_has_key(); } if (from.has_value()) { - ValueTypeHandler::EnsureMutable(&value_, GetArenaNoVirtual()); - ValueTypeHandler::Merge(from.value(), &value_, GetArenaNoVirtual()); + ValueTypeHandler::EnsureMutable(&value_, Base::GetArena()); + ValueTypeHandler::Merge(from.value(), &value_, Base::GetArena()); set_has_value(); } } } public: - void Clear() { - KeyTypeHandler::Clear(&key_, GetArenaNoVirtual()); - ValueTypeHandler::ClearMaybeByDefaultEnum( - &value_, GetArenaNoVirtual(), default_enum_value); + void Clear() override { + KeyTypeHandler::Clear(&key_, Base::GetArena()); + ValueTypeHandler::ClearMaybeByDefaultEnum(&value_, Base::GetArena(), + default_enum_value); clear_has_key(); clear_has_value(); } @@ -312,35 +342,20 @@ class MapEntryImpl : public Base { ValueTypeHandler::AssignDefaultValue(&d->value_); } - Arena* GetArena() const { - return GetArenaNoVirtual(); - } - - // Create a MapEntryImpl for given key and value from google::protobuf::Map in - // serialization. This function is only called when value is enum. Enum is - // treated differently because its type in MapEntry is int and its type in - // google::protobuf::Map is enum. We cannot create a reference to int from an enum. - static Derived* EnumWrap(const Key& key, const Value value, Arena* arena) { - return Arena::CreateMessage(arena, key, value); - } - - // Like above, but for all the other types. This avoids value copy to create - // MapEntryImpl from google::protobuf::Map in serialization. - static Derived* Wrap(const Key& key, const Value& value, Arena* arena) { - return Arena::CreateMessage(arena, key, value); - } - // Parsing using MergePartialFromCodedStream, above, is not as // efficient as it could be. This helper class provides a speedier way. template class Parser { public: explicit Parser(MapField* mf) : mf_(mf), map_(mf->MutableMap()) {} + ~Parser() { + if (entry_ != nullptr && entry_->GetArena() == nullptr) delete entry_; + } // This does what the typical MergePartialFromCodedStream() is expected to // do, with the additional side-effect that if successful (i.e., if true is // going to be its return value) it inserts the key-value pair into map_. - bool MergePartialFromCodedStream(::google::protobuf::io::CodedInputStream* input) { + bool MergePartialFromCodedStream(io::CodedInputStream* input) { // Look for the expected thing: a key and then a value. If it fails, // invoke the enclosing class's MergePartialFromCodedStream, or return // false if that would be pointless. @@ -353,11 +368,11 @@ class MapEntryImpl : public Base { int size; input->GetDirectBufferPointerInline(&data, &size); // We could use memcmp here, but we don't bother. The tag is one byte. - GOOGLE_COMPILE_ASSERT(kTagSize == 1, tag_size_error); + static_assert(kTagSize == 1, "tag size must be 1"); if (size > 0 && *reinterpret_cast(data) == kValueTag) { typename Map::size_type map_size = map_->size(); value_ptr_ = &(*map_)[key_]; - if (GOOGLE_PREDICT_TRUE(map_size != map_->size())) { + if (PROTOBUF_PREDICT_TRUE(map_size != map_->size())) { // We created a new key-value pair. Fill in the value. typedef typename MapIf::type T; @@ -375,64 +390,118 @@ class MapEntryImpl : public Base { key_ = Key(); } - entry_.reset(mf_->NewEntry()); + NewEntry(); *entry_->mutable_key() = key_; const bool result = entry_->MergePartialFromCodedStream(input); if (result) UseKeyAndValueFromEntry(); - if (entry_->GetArena() != NULL) entry_.release(); return result; } + const char* _InternalParse(const char* ptr, ParseContext* ctx) { + if (PROTOBUF_PREDICT_TRUE(!ctx->Done(&ptr) && *ptr == kKeyTag)) { + ptr = KeyTypeHandler::Read(ptr + 1, ctx, &key_); + if (PROTOBUF_PREDICT_FALSE(!ptr || !Derived::ValidateKey(&key_))) { + return nullptr; + } + if (PROTOBUF_PREDICT_TRUE(!ctx->Done(&ptr) && *ptr == kValueTag)) { + typename Map::size_type map_size = map_->size(); + value_ptr_ = &(*map_)[key_]; + if (PROTOBUF_PREDICT_TRUE(map_size != map_->size())) { + using T = + typename MapIf::type; + ptr = ValueTypeHandler::Read(ptr + 1, ctx, + reinterpret_cast(value_ptr_)); + if (PROTOBUF_PREDICT_FALSE(!ptr || + !Derived::ValidateValue(value_ptr_))) { + map_->erase(key_); // Failure! Undo insertion. + return nullptr; + } + if (PROTOBUF_PREDICT_TRUE(ctx->Done(&ptr))) return ptr; + if (!ptr) return nullptr; + NewEntry(); + ValueMover::Move(value_ptr_, entry_->mutable_value()); + map_->erase(key_); + goto move_key; + } + } else { + if (!ptr) return nullptr; + } + NewEntry(); + move_key: + KeyMover::Move(&key_, entry_->mutable_key()); + } else { + if (!ptr) return nullptr; + NewEntry(); + } + ptr = entry_->_InternalParse(ptr, ctx); + if (ptr) UseKeyAndValueFromEntry(); + return ptr; + } + + template + const char* ParseWithEnumValidation(const char* ptr, ParseContext* ctx, + bool (*is_valid)(int), uint32 field_num, + InternalMetadata* metadata) { + auto entry = NewEntry(); + ptr = entry->_InternalParse(ptr, ctx); + if (!ptr) return nullptr; + if (is_valid(entry->value())) { + UseKeyAndValueFromEntry(); + } else { + WriteLengthDelimited(field_num, entry->SerializeAsString(), + metadata->mutable_unknown_fields()); + } + return ptr; + } + + MapEntryImpl* NewEntry() { return entry_ = mf_->NewEntry(); } + const Key& key() const { return key_; } const Value& value() const { return *value_ptr_; } + const Key& entry_key() const { return entry_->key(); } + const Value& entry_value() const { return entry_->value(); } + private: - void UseKeyAndValueFromEntry() GOOGLE_PROTOBUF_ATTRIBUTE_COLD { + void UseKeyAndValueFromEntry() { // Update key_ in case we need it later (because key() is called). // This is potentially inefficient, especially if the key is // expensive to copy (e.g., a long string), but this is a cold // path, so it's not a big deal. key_ = entry_->key(); value_ptr_ = &(*map_)[key_]; - MoveHelper::Move(entry_->mutable_value(), value_ptr_); + ValueMover::Move(entry_->mutable_value(), value_ptr_); } // After reading a key and value successfully, and inserting that data // into map_, we are not at the end of the input. This is unusual, but // allowed by the spec. - bool ReadBeyondKeyValuePair(::google::protobuf::io::CodedInputStream* input) - GOOGLE_PROTOBUF_ATTRIBUTE_COLD { - typedef MoveHelper KeyMover; - typedef MoveHelper ValueMover; - entry_.reset(mf_->NewEntry()); + bool ReadBeyondKeyValuePair(io::CodedInputStream* input) PROTOBUF_COLD { + NewEntry(); ValueMover::Move(value_ptr_, entry_->mutable_value()); map_->erase(key_); KeyMover::Move(&key_, entry_->mutable_key()); const bool result = entry_->MergePartialFromCodedStream(input); if (result) UseKeyAndValueFromEntry(); - if (entry_->GetArena() != NULL) entry_.release(); return result; } + typedef MoveHelper + KeyMover; + typedef MoveHelper + ValueMover; + MapField* const mf_; Map* const map_; Key key_; Value* value_ptr_; - // On the fast path entry_ is not used. And, when entry_ is used, it's set - // to mf_->NewEntry(), so in the arena case we must call entry_.release. - std::unique_ptr entry_; + MapEntryImpl* entry_ = nullptr; }; protected: @@ -443,82 +512,16 @@ class MapEntryImpl : public Base { bool has_value() const { return (_has_bits_[0] & 0x00000002u) != 0; } void clear_has_value() { _has_bits_[0] &= ~0x00000002u; } - private: - // Serializing a generated message containing map field involves serializing - // key-value pairs from google::protobuf::Map. The wire format of each key-value pair - // after serialization should be the same as that of a MapEntry message - // containing the same key and value inside it. However, google::protobuf::Map doesn't - // store key and value as MapEntry message, which disables us to use existing - // code to serialize message. In order to use existing code to serialize - // message, we need to construct a MapEntry from key-value pair. But it - // involves copy of key and value to construct a MapEntry. In order to avoid - // this copy in constructing a MapEntry, we need the following class which - // only takes references of given key and value. - class MapEntryWrapper : public Derived { - typedef Derived BaseClass; - typedef typename BaseClass::KeyMapEntryAccessorType KeyMapEntryAccessorType; - typedef - typename BaseClass::ValueMapEntryAccessorType ValueMapEntryAccessorType; - - public: - MapEntryWrapper(Arena* arena, const Key& key, const Value& value) - : Derived(arena), key_(key), value_(value) { - BaseClass::set_has_key(); - BaseClass::set_has_value(); - } - inline const KeyMapEntryAccessorType& key() const { return key_; } - inline const ValueMapEntryAccessorType& value() const { return value_; } - - private: - const Key& key_; - const Value& value_; - - friend class ::google::protobuf::Arena; - typedef void InternalArenaConstructable_; - typedef void DestructorSkippable_; - }; - - // Like above, but for enum value only, which stores value instead of - // reference of value field inside. This is needed because the type of value - // field in constructor is an enum, while we need to store it as an int. If we - // initialize a reference to int with a reference to enum, compiler will - // generate a temporary int from enum and initialize the reference to int with - // the temporary. - class MapEnumEntryWrapper : public Derived { - typedef Derived BaseClass; - typedef typename BaseClass::KeyMapEntryAccessorType KeyMapEntryAccessorType; - typedef - typename BaseClass::ValueMapEntryAccessorType ValueMapEntryAccessorType; - - public: - MapEnumEntryWrapper(Arena* arena, const Key& key, const Value& value) - : Derived(arena), key_(key), value_(value) { - BaseClass::set_has_key(); - BaseClass::set_has_value(); - } - inline const KeyMapEntryAccessorType& key() const { return key_; } - inline const ValueMapEntryAccessorType& value() const { return value_; } - - private: - const KeyMapEntryAccessorType& key_; - const ValueMapEntryAccessorType value_; - - friend class google::protobuf::Arena; - typedef void DestructorSkippable_; - }; - - inline Arena* GetArenaNoVirtual() const { - return arena_; - } + public: + inline Arena* GetArena() const { return Base::GetArena(); } public: // Needed for constructing tables KeyOnMemory key_; ValueOnMemory value_; - Arena* arena_; uint32 _has_bits_[1]; private: - friend class ::google::protobuf::Arena; + friend class ::PROTOBUF_NAMESPACE_ID::Arena; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; template (); } void MergeFrom(const MapEntryLite& other) { MergeFromInternal(other); } private: @@ -570,7 +574,8 @@ struct DeconstructMapEntry > { // type is relatively small and easy to copy then copying Keys into an // array of SortItems can be beneficial. Then all the data the sorting // algorithm needs to touch is in that one array. -template struct SortItem { +template +struct SortItem { SortItem() {} explicit SortItem(PtrToKeyValuePair p) : first(p->first), second(p) {} @@ -578,16 +583,14 @@ template struct SortItem { PtrToKeyValuePair second; }; -template struct CompareByFirstField { - bool operator()(const T& a, const T& b) const { - return a.first < b.first; - } +template +struct CompareByFirstField { + bool operator()(const T& a, const T& b) const { return a.first < b.first; } }; -template struct CompareByDerefFirst { - bool operator()(const T& a, const T& b) const { - return a->first < b->first; - } +template +struct CompareByDerefFirst { + bool operator()(const T& a, const T& b) const { return a->first < b->first; } }; // Helper for table driven serialization @@ -602,20 +605,20 @@ struct FromHelper { template <> struct FromHelper { - static ArenaStringPtr From(const string& x) { + static ArenaStringPtr From(const std::string& x) { ArenaStringPtr res; - TaggedPtr<::std::string> ptr; - ptr.Set(const_cast(&x)); + TaggedPtr ptr; + ptr.Set(const_cast(&x)); res.UnsafeSetTaggedPointer(ptr); return res; } }; template <> struct FromHelper { - static ArenaStringPtr From(const string& x) { + static ArenaStringPtr From(const std::string& x) { ArenaStringPtr res; - TaggedPtr<::std::string> ptr; - ptr.Set(const_cast(&x)); + TaggedPtr ptr; + ptr.Set(const_cast(&x)); res.UnsafeSetTaggedPointer(ptr); return res; } @@ -653,7 +656,7 @@ struct MapEntryHelper::From(map_pair.first)), value_(FromHelper::From(map_pair.second)) {} - // Purposely not folowing the style guide naming. These are the names + // Purposely not following the style guide naming. These are the names // the proto compiler would generate given the map entry descriptor. // The proto compiler generates the offsets in this struct as if this was // a regular message. This way the table driven code barely notices it's @@ -666,6 +669,8 @@ struct MapEntryHelper + #endif // GOOGLE_PROTOBUF_MAP_ENTRY_LITE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/map_field_lite.h b/third_party/protobuf-lite/google/protobuf/map_field_lite.h index f648b430..d641d173 100644 --- a/third_party/protobuf-lite/google/protobuf/map_field_lite.h +++ b/third_party/protobuf-lite/google/protobuf/map_field_lite.h @@ -31,10 +31,20 @@ #ifndef GOOGLE_PROTOBUF_MAP_FIELD_LITE_H__ #define GOOGLE_PROTOBUF_MAP_FIELD_LITE_H__ +#include +#include +#include #include #include +#include #include +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + namespace google { namespace protobuf { namespace internal { @@ -53,11 +63,9 @@ class MapFieldLite { typedef Map MapType; typedef EntryType EntryTypeTrait; - MapFieldLite() : arena_(NULL) { SetDefaultEnumValue(); } + MapFieldLite() { SetDefaultEnumValue(); } - explicit MapFieldLite(Arena* arena) : arena_(arena), map_(arena) { - SetDefaultEnumValue(); - } + explicit MapFieldLite(Arena* arena) : map_(arena) { SetDefaultEnumValue(); } // Accessors const Map& GetMap() const { return map_; } @@ -82,38 +90,75 @@ class MapFieldLite { // Used in the implementation of parsing. Caller should take the ownership iff // arena_ is NULL. EntryType* NewEntry() const { - if (arena_ == NULL) { - return new EntryType(); - } else { - return Arena::CreateMessage(arena_); - } + return Arena::CreateMessage(map_.arena_); } // Used in the implementation of serializing enum value type. Caller should // take the ownership iff arena_ is NULL. EntryType* NewEnumEntryWrapper(const Key& key, const T t) const { - return EntryType::EnumWrap(key, t, arena_); + return EntryType::EnumWrap(key, t, map_.arena_); } // Used in the implementation of serializing other value types. Caller should // take the ownership iff arena_ is NULL. EntryType* NewEntryWrapper(const Key& key, const T& t) const { - return EntryType::Wrap(key, t, arena_); + return EntryType::Wrap(key, t, map_.arena_); + } + + const char* _InternalParse(const char* ptr, ParseContext* ctx) { + typename Derived::template Parser> parser(this); + return parser._InternalParse(ptr, ctx); + } + + template + const char* ParseWithEnumValidation(const char* ptr, ParseContext* ctx, + bool (*is_valid)(int), uint32 field_num, + InternalMetadata* metadata) { + typename Derived::template Parser> parser(this); + return parser.template ParseWithEnumValidation( + ptr, ctx, is_valid, field_num, metadata); } private: typedef void DestructorSkippable_; - Arena* arena_; Map map_; - friend class ::google::protobuf::Arena; + friend class ::PROTOBUF_NAMESPACE_ID::Arena; +}; + +template +struct EnumParseWrapper { + const char* _InternalParse(const char* ptr, ParseContext* ctx) { + return map_field->template ParseWithEnumValidation( + ptr, ctx, is_valid, field_num, metadata); + } + T* map_field; + bool (*is_valid)(int); + uint32 field_num; + InternalMetadata* metadata; }; +// Helper function because the typenames of maps are horrendous to print. This +// leverages compiler type deduction, to keep all type data out of the +// generated code +template +EnumParseWrapper InitEnumParseWrapper( + T* map_field, bool (*is_valid)(int), uint32 field_num, + InternalMetadata* metadata) { + return EnumParseWrapper{map_field, is_valid, field_num, + metadata}; +} + // True if IsInitialized() is true for value field in all elements of t. T is // expected to be message. It's useful to have this helper here to keep the // protobuf compiler from ever having to emit loops in IsInitialized() methods. // We want the C++ compiler to inline this or not as it sees fit. -template -bool AllAreInitialized(const Map& t) { +template +bool AllAreInitialized( + const MapFieldLite& field) { + const auto& t = field.GetMap(); for (typename Map::const_iterator it = t.begin(); it != t.end(); ++it) { if (!it->second.IsInitialized()) return false; @@ -128,7 +173,7 @@ template struct MapEntryToMapField > { + kValueFieldType, default_enum_value>> { typedef MapFieldLite, Key, Value, kKeyFieldType, kValueFieldType, @@ -138,6 +183,8 @@ struct MapEntryToMapField + #endif // GOOGLE_PROTOBUF_MAP_FIELD_LITE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/map_type_handler.h b/third_party/protobuf-lite/google/protobuf/map_type_handler.h index 7f7b1e0e..5efc6fb6 100644 --- a/third_party/protobuf-lite/google/protobuf/map_type_handler.h +++ b/third_party/protobuf-lite/google/protobuf/map_type_handler.h @@ -31,8 +31,14 @@ #ifndef GOOGLE_PROTOBUF_TYPE_HANDLER_H__ #define GOOGLE_PROTOBUF_TYPE_HANDLER_H__ +#include +#include #include -#include +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { @@ -40,15 +46,15 @@ namespace internal { // Used for compile time type selection. MapIf::type will be TrueType if Flag is // true and FalseType otherwise. -template +template struct MapIf; -template +template struct MapIf { typedef TrueType type; }; -template +template struct MapIf { typedef FalseType type; }; @@ -72,7 +78,8 @@ class MapValueInitializer { template class MapValueInitializer { public: - static inline void Initialize(Type& /* value */, int /* default_enum_value */) {} + static inline void Initialize(Type& /* value */, + int /* default_enum_value */) {} }; template @@ -113,23 +120,23 @@ class MapWireFieldTypeTraits {}; WireFormatLite::WIRETYPE_##WireFormatType; \ }; -TYPE_TRAITS(MESSAGE , Type, LENGTH_DELIMITED, true, false) -TYPE_TRAITS(STRING , ArenaStringPtr, LENGTH_DELIMITED, false, false) -TYPE_TRAITS(BYTES , ArenaStringPtr , LENGTH_DELIMITED, false, false) -TYPE_TRAITS(INT64 , int64 , VARINT , false, false) -TYPE_TRAITS(UINT64 , uint64 , VARINT , false, false) -TYPE_TRAITS(INT32 , int32 , VARINT , false, false) -TYPE_TRAITS(UINT32 , uint32 , VARINT , false, false) -TYPE_TRAITS(SINT64 , int64 , VARINT , false, false) -TYPE_TRAITS(SINT32 , int32 , VARINT , false, false) -TYPE_TRAITS(ENUM , int , VARINT , false, true ) -TYPE_TRAITS(DOUBLE , double , FIXED64, false, false) -TYPE_TRAITS(FLOAT , float , FIXED32, false, false) -TYPE_TRAITS(FIXED64 , uint64 , FIXED64, false, false) -TYPE_TRAITS(FIXED32 , uint32 , FIXED32, false, false) -TYPE_TRAITS(SFIXED64, int64 , FIXED64, false, false) -TYPE_TRAITS(SFIXED32, int32 , FIXED32, false, false) -TYPE_TRAITS(BOOL , bool , VARINT , false, false) +TYPE_TRAITS(MESSAGE, Type, LENGTH_DELIMITED, true, false) +TYPE_TRAITS(STRING, ArenaStringPtr, LENGTH_DELIMITED, false, false) +TYPE_TRAITS(BYTES, ArenaStringPtr, LENGTH_DELIMITED, false, false) +TYPE_TRAITS(INT64, int64, VARINT, false, false) +TYPE_TRAITS(UINT64, uint64, VARINT, false, false) +TYPE_TRAITS(INT32, int32, VARINT, false, false) +TYPE_TRAITS(UINT32, uint32, VARINT, false, false) +TYPE_TRAITS(SINT64, int64, VARINT, false, false) +TYPE_TRAITS(SINT32, int32, VARINT, false, false) +TYPE_TRAITS(ENUM, int, VARINT, false, true) +TYPE_TRAITS(DOUBLE, double, FIXED64, false, false) +TYPE_TRAITS(FLOAT, float, FIXED32, false, false) +TYPE_TRAITS(FIXED64, uint64, FIXED64, false, false) +TYPE_TRAITS(FIXED32, uint32, FIXED32, false, false) +TYPE_TRAITS(SFIXED64, int64, FIXED64, false, false) +TYPE_TRAITS(SFIXED32, int32, FIXED32, false, false) +TYPE_TRAITS(BOOL, bool, VARINT, false, false) #undef TYPE_TRAITS @@ -142,18 +149,19 @@ class MapTypeHandler { // Enum type cannot be used for MapTypeHandler::Read. Define a type which will // replace Enum with int. typedef typename MapWireFieldTypeTraits::MapEntryAccessorType MapEntryAccessorType; + Type>::MapEntryAccessorType + MapEntryAccessorType; // Internal stored type in MapEntryLite for given wire field type. typedef typename MapWireFieldTypeTraits::TypeOnMemory TypeOnMemory; // Corresponding wire type for field type. - static const WireFormatLite::WireType kWireType = + static constexpr WireFormatLite::WireType kWireType = MapWireFieldTypeTraits::kWireType; // Whether wire type is for message. - static const bool kIsMessage = + static constexpr bool kIsMessage = MapWireFieldTypeTraits::kIsMessage; // Whether wire type is for enum. - static const bool kIsEnum = + static constexpr bool kIsEnum = MapWireFieldTypeTraits::kIsEnum; // Functions used in parsing and serialization. =================== @@ -161,14 +169,11 @@ class MapTypeHandler { static inline int GetCachedSize(const MapEntryAccessorType& value); static inline bool Read(io::CodedInputStream* input, MapEntryAccessorType* value); - static inline void Write(int field, const MapEntryAccessorType& value, - io::CodedOutputStream* output); - static inline uint8* InternalWriteToArray(int field, - const MapEntryAccessorType& value, - bool deterministic, uint8* target); - static inline uint8* WriteToArray(int field, - const MapEntryAccessorType& value, - uint8* target); + static inline const char* Read(const char* ptr, ParseContext* ctx, + MapEntryAccessorType* value); + + static inline uint8* Write(int field, const MapEntryAccessorType& value, + uint8* ptr, io::EpsCopyOutputStream* stream); // Functions to manipulate data on memory. ======================== static inline const Type& GetExternalReference(const Type* value); @@ -192,8 +197,8 @@ class MapTypeHandler { static inline void AssignDefaultValue(Type** value); // Return default instance if value is not initialized when calling const // reference accessor. - static inline const Type& DefaultIfNotInitialized( - const Type* value, const Type* default_value); + static inline const Type& DefaultIfNotInitialized(const Type* value, + const Type* default_value); // Check if all required fields have values set. static inline bool IsInitialized(Type* value); }; @@ -220,16 +225,10 @@ class MapTypeHandler { static inline int GetCachedSize(const MapEntryAccessorType& value); \ static inline bool Read(io::CodedInputStream* input, \ MapEntryAccessorType* value); \ - static inline void Write(int field, const MapEntryAccessorType& value, \ - io::CodedOutputStream* output); \ - static inline uint8* InternalWriteToArray( \ - int field, const MapEntryAccessorType& value, bool deterministic, \ - uint8* target); \ - static inline uint8* WriteToArray(int field, \ - const MapEntryAccessorType& value, \ - uint8* target) { \ - return InternalWriteToArray(field, value, false, target); \ - } \ + static inline const char* Read(const char* begin, ParseContext* ctx, \ + MapEntryAccessorType* value); \ + static inline uint8* Write(int field, const MapEntryAccessorType& value, \ + uint8* ptr, io::EpsCopyOutputStream* stream); \ static inline const MapEntryAccessorType& GetExternalReference( \ const TypeOnMemory& value); \ static inline void DeleteNoArena(const TypeOnMemory& x); \ @@ -241,7 +240,7 @@ class MapTypeHandler { int default_enum); \ static inline size_t SpaceUsedInMapEntryLong(const TypeOnMemory& value); \ static inline size_t SpaceUsedInMapLong(const TypeOnMemory& value); \ - static inline size_t SpaceUsedInMapLong(const string& value); \ + static inline size_t SpaceUsedInMapLong(const std::string& value); \ static inline void AssignDefaultValue(TypeOnMemory* value); \ static inline const MapEntryAccessorType& DefaultIfNotInitialized( \ const TypeOnMemory& value, const TypeOnMemory& default_value); \ @@ -273,8 +272,7 @@ MAP_HANDLER(BOOL) #undef MAP_HANDLER template -inline size_t -MapTypeHandler::ByteSize( +inline size_t MapTypeHandler::ByteSize( const MapEntryAccessorType& value) { return WireFormatLite::MessageSizeNoVirtual(value); } @@ -287,14 +285,14 @@ MapTypeHandler::ByteSize( } GOOGLE_PROTOBUF_BYTE_SIZE(STRING, String) -GOOGLE_PROTOBUF_BYTE_SIZE(BYTES , Bytes) -GOOGLE_PROTOBUF_BYTE_SIZE(INT64 , Int64) +GOOGLE_PROTOBUF_BYTE_SIZE(BYTES, Bytes) +GOOGLE_PROTOBUF_BYTE_SIZE(INT64, Int64) GOOGLE_PROTOBUF_BYTE_SIZE(UINT64, UInt64) -GOOGLE_PROTOBUF_BYTE_SIZE(INT32 , Int32) +GOOGLE_PROTOBUF_BYTE_SIZE(INT32, Int32) GOOGLE_PROTOBUF_BYTE_SIZE(UINT32, UInt32) GOOGLE_PROTOBUF_BYTE_SIZE(SINT64, SInt64) GOOGLE_PROTOBUF_BYTE_SIZE(SINT32, SInt32) -GOOGLE_PROTOBUF_BYTE_SIZE(ENUM , Enum) +GOOGLE_PROTOBUF_BYTE_SIZE(ENUM, Enum) #undef GOOGLE_PROTOBUF_BYTE_SIZE @@ -305,23 +303,21 @@ GOOGLE_PROTOBUF_BYTE_SIZE(ENUM , Enum) return WireFormatLite::k##DeclaredType##Size; \ } -FIXED_BYTE_SIZE(DOUBLE , Double) -FIXED_BYTE_SIZE(FLOAT , Float) -FIXED_BYTE_SIZE(FIXED64 , Fixed64) -FIXED_BYTE_SIZE(FIXED32 , Fixed32) +FIXED_BYTE_SIZE(DOUBLE, Double) +FIXED_BYTE_SIZE(FLOAT, Float) +FIXED_BYTE_SIZE(FIXED64, Fixed64) +FIXED_BYTE_SIZE(FIXED32, Fixed32) FIXED_BYTE_SIZE(SFIXED64, SFixed64) FIXED_BYTE_SIZE(SFIXED32, SFixed32) -FIXED_BYTE_SIZE(BOOL , Bool) +FIXED_BYTE_SIZE(BOOL, Bool) #undef FIXED_BYTE_SIZE template -inline int -MapTypeHandler::GetCachedSize( +inline int MapTypeHandler::GetCachedSize( const MapEntryAccessorType& value) { - return static_cast( - WireFormatLite::LengthDelimitedSize( - static_cast(value.GetCachedSize()))); + return static_cast(WireFormatLite::LengthDelimitedSize( + static_cast(value.GetCachedSize()))); } #define GET_CACHED_SIZE(FieldType, DeclaredType) \ @@ -333,14 +329,14 @@ MapTypeHandler::GetCachedSize( } GET_CACHED_SIZE(STRING, String) -GET_CACHED_SIZE(BYTES , Bytes) -GET_CACHED_SIZE(INT64 , Int64) +GET_CACHED_SIZE(BYTES, Bytes) +GET_CACHED_SIZE(INT64, Int64) GET_CACHED_SIZE(UINT64, UInt64) -GET_CACHED_SIZE(INT32 , Int32) +GET_CACHED_SIZE(INT32, Int32) GET_CACHED_SIZE(UINT32, UInt32) GET_CACHED_SIZE(SINT64, SInt64) GET_CACHED_SIZE(SINT32, SInt32) -GET_CACHED_SIZE(ENUM , Enum) +GET_CACHED_SIZE(ENUM, Enum) #undef GET_CACHED_SIZE @@ -352,63 +348,60 @@ GET_CACHED_SIZE(ENUM , Enum) return WireFormatLite::k##DeclaredType##Size; \ } -GET_FIXED_CACHED_SIZE(DOUBLE , Double) -GET_FIXED_CACHED_SIZE(FLOAT , Float) -GET_FIXED_CACHED_SIZE(FIXED64 , Fixed64) -GET_FIXED_CACHED_SIZE(FIXED32 , Fixed32) +GET_FIXED_CACHED_SIZE(DOUBLE, Double) +GET_FIXED_CACHED_SIZE(FLOAT, Float) +GET_FIXED_CACHED_SIZE(FIXED64, Fixed64) +GET_FIXED_CACHED_SIZE(FIXED32, Fixed32) GET_FIXED_CACHED_SIZE(SFIXED64, SFixed64) GET_FIXED_CACHED_SIZE(SFIXED32, SFixed32) -GET_FIXED_CACHED_SIZE(BOOL , Bool) +GET_FIXED_CACHED_SIZE(BOOL, Bool) #undef GET_FIXED_CACHED_SIZE template -inline void MapTypeHandler::Write( - int field, const MapEntryAccessorType& value, - io::CodedOutputStream* output) { - WireFormatLite::WriteMessageMaybeToArray(field, value, output); -} - -template -inline uint8* -MapTypeHandler::InternalWriteToArray( - int field, const MapEntryAccessorType& value, bool deterministic, - uint8* target) { - return WireFormatLite::InternalWriteMessageToArray(field, value, - deterministic, target); +inline uint8* MapTypeHandler::Write( + int field, const MapEntryAccessorType& value, uint8* ptr, + io::EpsCopyOutputStream* stream) { + ptr = stream->EnsureSpace(ptr); + return WireFormatLite::InternalWriteMessage(field, value, ptr, stream); } #define WRITE_METHOD(FieldType, DeclaredType) \ template \ - inline void MapTypeHandler::Write( \ - int field, const MapEntryAccessorType& value, \ - io::CodedOutputStream* output) { \ - return WireFormatLite::Write##DeclaredType(field, value, output); \ - } \ + inline uint8* MapTypeHandler::Write( \ + int field, const MapEntryAccessorType& value, uint8* ptr, \ + io::EpsCopyOutputStream* stream) { \ + ptr = stream->EnsureSpace(ptr); \ + return stream->Write##DeclaredType(field, value, ptr); \ + } + +WRITE_METHOD(STRING, String) +WRITE_METHOD(BYTES, Bytes) + +#undef WRITE_METHOD +#define WRITE_METHOD(FieldType, DeclaredType) \ template \ - inline uint8* \ - MapTypeHandler::InternalWriteToArray( \ - int field, const MapEntryAccessorType& value, bool, uint8* target) { \ - return WireFormatLite::Write##DeclaredType##ToArray(field, value, target); \ + inline uint8* MapTypeHandler::Write( \ + int field, const MapEntryAccessorType& value, uint8* ptr, \ + io::EpsCopyOutputStream* stream) { \ + ptr = stream->EnsureSpace(ptr); \ + return WireFormatLite::Write##DeclaredType##ToArray(field, value, ptr); \ } -WRITE_METHOD(STRING , String) -WRITE_METHOD(BYTES , Bytes) -WRITE_METHOD(INT64 , Int64) -WRITE_METHOD(UINT64 , UInt64) -WRITE_METHOD(INT32 , Int32) -WRITE_METHOD(UINT32 , UInt32) -WRITE_METHOD(SINT64 , SInt64) -WRITE_METHOD(SINT32 , SInt32) -WRITE_METHOD(ENUM , Enum) -WRITE_METHOD(DOUBLE , Double) -WRITE_METHOD(FLOAT , Float) -WRITE_METHOD(FIXED64 , Fixed64) -WRITE_METHOD(FIXED32 , Fixed32) +WRITE_METHOD(INT64, Int64) +WRITE_METHOD(UINT64, UInt64) +WRITE_METHOD(INT32, Int32) +WRITE_METHOD(UINT32, UInt32) +WRITE_METHOD(SINT64, SInt64) +WRITE_METHOD(SINT32, SInt32) +WRITE_METHOD(ENUM, Enum) +WRITE_METHOD(DOUBLE, Double) +WRITE_METHOD(FLOAT, Float) +WRITE_METHOD(FIXED64, Fixed64) +WRITE_METHOD(FIXED32, Fixed32) WRITE_METHOD(SFIXED64, SFixed64) WRITE_METHOD(SFIXED32, SFixed32) -WRITE_METHOD(BOOL , Bool) +WRITE_METHOD(BOOL, Bool) #undef WRITE_METHOD @@ -430,6 +423,82 @@ inline bool MapTypeHandler::Read( return WireFormatLite::ReadBytes(input, value); } +template +const char* MapTypeHandler::Read( + const char* ptr, ParseContext* ctx, MapEntryAccessorType* value) { + return ctx->ParseMessage(value, ptr); +} + +template +const char* MapTypeHandler::Read( + const char* ptr, ParseContext* ctx, MapEntryAccessorType* value) { + int size = ReadSize(&ptr); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + return ctx->ReadString(ptr, size, value); +} + +template +const char* MapTypeHandler::Read( + const char* ptr, ParseContext* ctx, MapEntryAccessorType* value) { + int size = ReadSize(&ptr); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + return ctx->ReadString(ptr, size, value); +} + +inline const char* ReadINT64(const char* ptr, int64* value) { + return VarintParse(ptr, reinterpret_cast(value)); +} +inline const char* ReadUINT64(const char* ptr, uint64* value) { + return VarintParse(ptr, value); +} +inline const char* ReadINT32(const char* ptr, int32* value) { + return VarintParse(ptr, reinterpret_cast(value)); +} +inline const char* ReadUINT32(const char* ptr, uint32* value) { + return VarintParse(ptr, value); +} +inline const char* ReadSINT64(const char* ptr, int64* value) { + *value = ReadVarintZigZag64(&ptr); + return ptr; +} +inline const char* ReadSINT32(const char* ptr, int32* value) { + *value = ReadVarintZigZag32(&ptr); + return ptr; +} +template +inline const char* ReadENUM(const char* ptr, E* value) { + *value = static_cast(ReadVarint32(&ptr)); + return ptr; +} +inline const char* ReadBOOL(const char* ptr, bool* value) { + *value = static_cast(ReadVarint32(&ptr)); + return ptr; +} + +template +inline const char* ReadUnaligned(const char* ptr, F* value) { + *value = UnalignedLoad(ptr); + return ptr + sizeof(F); +} +inline const char* ReadFLOAT(const char* ptr, float* value) { + return ReadUnaligned(ptr, value); +} +inline const char* ReadDOUBLE(const char* ptr, double* value) { + return ReadUnaligned(ptr, value); +} +inline const char* ReadFIXED64(const char* ptr, uint64* value) { + return ReadUnaligned(ptr, value); +} +inline const char* ReadFIXED32(const char* ptr, uint32* value) { + return ReadUnaligned(ptr, value); +} +inline const char* ReadSFIXED64(const char* ptr, int64* value) { + return ReadUnaligned(ptr, value); +} +inline const char* ReadSFIXED32(const char* ptr, int32* value) { + return ReadUnaligned(ptr, value); +} + #define READ_METHOD(FieldType) \ template \ inline bool MapTypeHandler::Read( \ @@ -437,6 +506,12 @@ inline bool MapTypeHandler::Read( return WireFormatLite::ReadPrimitive( \ input, value); \ + } \ + template \ + const char* MapTypeHandler::Read( \ + const char* begin, ParseContext* ctx, MapEntryAccessorType* value) { \ + (void)ctx; \ + return Read##FieldType(begin, value); \ } READ_METHOD(INT64) @@ -460,8 +535,8 @@ READ_METHOD(BOOL) template inline const Type& -MapTypeHandler::GetExternalReference(const Type* value) { +MapTypeHandler::GetExternalReference( + const Type* value) { return *value; } @@ -484,10 +559,8 @@ inline void MapTypeHandler::Clear( } template inline void -MapTypeHandler::ClearMaybeByDefaultEnum(Type** value, - Arena* /* arena */, - int /* default_enum_value */) { +MapTypeHandler::ClearMaybeByDefaultEnum( + Type** value, Arena* /* arena */, int /* default_enum_value */) { if (*value != NULL) (*value)->Clear(); } template @@ -504,14 +577,13 @@ void MapTypeHandler::DeleteNoArena( template inline void MapTypeHandler::AssignDefaultValue(Type** value) { + Type>::AssignDefaultValue(Type** value) { *value = const_cast(Type::internal_default_instance()); } template -inline void MapTypeHandler::Initialize(Type** x, - Arena* /* arena */) { +inline void MapTypeHandler::Initialize( + Type** x, Arena* /* arena */) { *x = NULL; } @@ -523,198 +595,194 @@ inline void MapTypeHandler:: } template -inline Type* MapTypeHandler::EnsureMutable(Type** value, - Arena* arena) { +inline Type* MapTypeHandler::EnsureMutable( + Type** value, Arena* arena) { if (*value == NULL) { - *value = - MapArenaMessageCreator:: - type::value>::CreateMessage(arena); + *value = MapArenaMessageCreator< + Type, + Arena::is_arena_constructable::type::value>::CreateMessage(arena); } return *value; } template -inline const Type& MapTypeHandler:: - DefaultIfNotInitialized(const Type* value, const Type* default_value) { +inline const Type& +MapTypeHandler::DefaultIfNotInitialized( + const Type* value, const Type* default_value) { return value != NULL ? *value : *default_value; } template -inline bool MapTypeHandler::IsInitialized(Type* value) { - return value->IsInitialized(); +inline bool MapTypeHandler::IsInitialized( + Type* value) { + return value ? value->IsInitialized() : false; } // Definition for string/bytes handler -#define STRING_OR_BYTES_HANDLER_FUNCTIONS(FieldType) \ - template \ - inline const typename MapTypeHandler::MapEntryAccessorType& \ - MapTypeHandler::GetExternalReference(const TypeOnMemory& value) { \ - return value.Get(); \ - } \ - template \ - inline size_t \ - MapTypeHandler::SpaceUsedInMapEntryLong(const TypeOnMemory& value) { \ - return sizeof(value); \ - } \ - template \ - inline size_t \ - MapTypeHandler::SpaceUsedInMapLong( \ - const TypeOnMemory& value) { \ - return sizeof(value); \ - } \ - template \ - inline size_t \ - MapTypeHandler::SpaceUsedInMapLong( \ - const string& value) { \ - return sizeof(value); \ - } \ - template \ - inline void MapTypeHandler::Clear( \ - TypeOnMemory* value, Arena* arena) { \ - value->ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), \ - arena); \ - } \ - template \ - inline void MapTypeHandler:: \ - ClearMaybeByDefaultEnum(TypeOnMemory* value, Arena* arena, \ - int /* default_enum */) { \ - Clear(value, arena); \ - } \ - template \ - inline void MapTypeHandler::Merge( \ - const MapEntryAccessorType& from, TypeOnMemory* to, Arena* arena) { \ - to->Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from, arena); \ - } \ - template \ - void MapTypeHandler::DeleteNoArena( \ - TypeOnMemory& value) { \ - value.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); \ - } \ - template \ - inline void MapTypeHandler::AssignDefaultValue(TypeOnMemory* /* value */) {} \ - template \ - inline void \ - MapTypeHandler::Initialize( \ - TypeOnMemory* value, Arena* /* arena */) { \ - value->UnsafeSetDefault( \ - &::google::protobuf::internal::GetEmptyStringAlreadyInited()); \ - } \ - template \ - inline void MapTypeHandler:: \ - InitializeMaybeByDefaultEnum(TypeOnMemory* value, \ - int /* default_enum_value */, \ - Arena* arena) { \ - Initialize(value, arena); \ - } \ - template \ - inline typename MapTypeHandler::MapEntryAccessorType* \ - MapTypeHandler::EnsureMutable( \ - TypeOnMemory* value, Arena* arena) { \ - return value->Mutable(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), \ - arena); \ - } \ - template \ - inline const typename MapTypeHandler::MapEntryAccessorType& \ - MapTypeHandler::DefaultIfNotInitialized(const TypeOnMemory& value, \ - const TypeOnMemory& \ - /* default_value */) { \ - return value.Get(); \ - } \ - template \ - inline bool MapTypeHandler::IsInitialized(const TypeOnMemory& /* value */) { \ - return true; \ +#define STRING_OR_BYTES_HANDLER_FUNCTIONS(FieldType) \ + template \ + inline const typename MapTypeHandler::MapEntryAccessorType& \ + MapTypeHandler::GetExternalReference(const TypeOnMemory& value) { \ + return value.Get(); \ + } \ + template \ + inline size_t \ + MapTypeHandler::SpaceUsedInMapEntryLong(const TypeOnMemory& value) { \ + return sizeof(value); \ + } \ + template \ + inline size_t \ + MapTypeHandler::SpaceUsedInMapLong( \ + const TypeOnMemory& value) { \ + return sizeof(value); \ + } \ + template \ + inline size_t \ + MapTypeHandler::SpaceUsedInMapLong( \ + const std::string& value) { \ + return sizeof(value); \ + } \ + template \ + inline void MapTypeHandler::Clear( \ + TypeOnMemory* value, Arena* arena) { \ + value->ClearToEmpty(&internal::GetEmptyStringAlreadyInited(), arena); \ + } \ + template \ + inline void MapTypeHandler:: \ + ClearMaybeByDefaultEnum(TypeOnMemory* value, Arena* arena, \ + int /* default_enum */) { \ + Clear(value, arena); \ + } \ + template \ + inline void MapTypeHandler::Merge( \ + const MapEntryAccessorType& from, TypeOnMemory* to, Arena* arena) { \ + to->Set(&internal::GetEmptyStringAlreadyInited(), from, arena); \ + } \ + template \ + void MapTypeHandler::DeleteNoArena( \ + TypeOnMemory& value) { \ + value.DestroyNoArena(&internal::GetEmptyStringAlreadyInited()); \ + } \ + template \ + inline void \ + MapTypeHandler::AssignDefaultValue( \ + TypeOnMemory* /* value */) {} \ + template \ + inline void \ + MapTypeHandler::Initialize( \ + TypeOnMemory* value, Arena* /* arena */) { \ + value->UnsafeSetDefault(&internal::GetEmptyStringAlreadyInited()); \ + } \ + template \ + inline void MapTypeHandler:: \ + InitializeMaybeByDefaultEnum( \ + TypeOnMemory* value, int /* default_enum_value */, Arena* arena) { \ + Initialize(value, arena); \ + } \ + template \ + inline typename MapTypeHandler::MapEntryAccessorType* \ + MapTypeHandler::EnsureMutable( \ + TypeOnMemory* value, Arena* arena) { \ + return value->Mutable(&internal::GetEmptyStringAlreadyInited(), arena); \ + } \ + template \ + inline const typename MapTypeHandler::MapEntryAccessorType& \ + MapTypeHandler:: \ + DefaultIfNotInitialized(const TypeOnMemory& value, \ + const TypeOnMemory& /* default_value */) { \ + return value.Get(); \ + } \ + template \ + inline bool \ + MapTypeHandler::IsInitialized( \ + const TypeOnMemory& /* value */) { \ + return true; \ } STRING_OR_BYTES_HANDLER_FUNCTIONS(STRING) STRING_OR_BYTES_HANDLER_FUNCTIONS(BYTES) #undef STRING_OR_BYTES_HANDLER_FUNCTIONS -#define PRIMITIVE_HANDLER_FUNCTIONS(FieldType) \ - template \ - inline const typename MapTypeHandler::MapEntryAccessorType& \ - MapTypeHandler::GetExternalReference(const TypeOnMemory& value) { \ - return value; \ - } \ - template \ - inline size_t \ - MapTypeHandler::SpaceUsedInMapEntryLong(const TypeOnMemory& /* value */) { \ - return 0; \ - } \ - template \ - inline size_t \ - MapTypeHandler::SpaceUsedInMapLong( \ - const TypeOnMemory& /* value */) { \ - return sizeof(Type); \ - } \ - template \ - inline void MapTypeHandler::Clear( \ - TypeOnMemory* value, Arena* /* arena */) { \ - *value = 0; \ - } \ - template \ - inline void MapTypeHandler:: \ - ClearMaybeByDefaultEnum(TypeOnMemory* value, Arena* /* arena */, \ - int default_enum_value) { \ - *value = static_cast(default_enum_value); \ - } \ - template \ - inline void MapTypeHandler::Merge( \ - const MapEntryAccessorType& from, TypeOnMemory* to, \ - Arena* /* arena */) { \ - *to = from; \ - } \ - template \ - inline void MapTypeHandler::DeleteNoArena(TypeOnMemory& /* x */) {} \ - template \ - inline void MapTypeHandler::AssignDefaultValue(TypeOnMemory* /* value */) {} \ - template \ - inline void \ - MapTypeHandler::Initialize( \ - TypeOnMemory* value, Arena* /* arena */) { \ - *value = 0; \ - } \ - template \ - inline void MapTypeHandler:: \ - InitializeMaybeByDefaultEnum(TypeOnMemory* value, \ - int default_enum_value, \ - Arena* /* arena */) { \ - *value = static_cast(default_enum_value); \ - } \ - template \ - inline typename MapTypeHandler::MapEntryAccessorType* \ - MapTypeHandler::EnsureMutable( \ - TypeOnMemory* value, Arena* /* arena */) { \ - return value; \ - } \ - template \ - inline const typename MapTypeHandler::MapEntryAccessorType& \ - MapTypeHandler::DefaultIfNotInitialized(const TypeOnMemory& value, \ - const TypeOnMemory& \ - /* default_value */) { \ - return value; \ - } \ - template \ - inline bool MapTypeHandler::IsInitialized(const TypeOnMemory& /* value */) { \ - return true; \ +#define PRIMITIVE_HANDLER_FUNCTIONS(FieldType) \ + template \ + inline const typename MapTypeHandler::MapEntryAccessorType& \ + MapTypeHandler::GetExternalReference(const TypeOnMemory& value) { \ + return value; \ + } \ + template \ + inline size_t MapTypeHandler:: \ + SpaceUsedInMapEntryLong(const TypeOnMemory& /* value */) { \ + return 0; \ + } \ + template \ + inline size_t \ + MapTypeHandler::SpaceUsedInMapLong( \ + const TypeOnMemory& /* value */) { \ + return sizeof(Type); \ + } \ + template \ + inline void MapTypeHandler::Clear( \ + TypeOnMemory* value, Arena* /* arena */) { \ + *value = 0; \ + } \ + template \ + inline void MapTypeHandler:: \ + ClearMaybeByDefaultEnum(TypeOnMemory* value, Arena* /* arena */, \ + int default_enum_value) { \ + *value = static_cast(default_enum_value); \ + } \ + template \ + inline void MapTypeHandler::Merge( \ + const MapEntryAccessorType& from, TypeOnMemory* to, \ + Arena* /* arena */) { \ + *to = from; \ + } \ + template \ + inline void MapTypeHandler::DeleteNoArena(TypeOnMemory& /* x */) {} \ + template \ + inline void \ + MapTypeHandler::AssignDefaultValue( \ + TypeOnMemory* /* value */) {} \ + template \ + inline void \ + MapTypeHandler::Initialize( \ + TypeOnMemory* value, Arena* /* arena */) { \ + *value = 0; \ + } \ + template \ + inline void MapTypeHandler:: \ + InitializeMaybeByDefaultEnum( \ + TypeOnMemory* value, int default_enum_value, Arena* /* arena */) { \ + *value = static_cast(default_enum_value); \ + } \ + template \ + inline typename MapTypeHandler::MapEntryAccessorType* \ + MapTypeHandler::EnsureMutable( \ + TypeOnMemory* value, Arena* /* arena */) { \ + return value; \ + } \ + template \ + inline const typename MapTypeHandler::MapEntryAccessorType& \ + MapTypeHandler:: \ + DefaultIfNotInitialized(const TypeOnMemory& value, \ + const TypeOnMemory& /* default_value */) { \ + return value; \ + } \ + template \ + inline bool \ + MapTypeHandler::IsInitialized( \ + const TypeOnMemory& /* value */) { \ + return true; \ } PRIMITIVE_HANDLER_FUNCTIONS(INT64) PRIMITIVE_HANDLER_FUNCTIONS(UINT64) @@ -734,6 +802,6 @@ PRIMITIVE_HANDLER_FUNCTIONS(BOOL) } // namespace internal } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_TYPE_HANDLER_H__ diff --git a/third_party/protobuf-lite/google/protobuf/message_lite.h b/third_party/protobuf-lite/google/protobuf/message_lite.h index b8644142..32a79bdb 100644 --- a/third_party/protobuf-lite/google/protobuf/message_lite.h +++ b/third_party/protobuf-lite/google/protobuf/message_lite.h @@ -40,29 +40,47 @@ #define GOOGLE_PROTOBUF_MESSAGE_LITE_H__ #include +#include + #include #include -#include +#include #include -#include +#include +#include +#include +#include + + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { + template class RepeatedPtrField; + namespace io { + class CodedInputStream; class CodedOutputStream; class ZeroCopyInputStream; class ZeroCopyOutputStream; -} + +} // namespace io namespace internal { +// See parse_context.h for explanation +class ParseContext; + class RepeatedPtrFieldBase; class WireFormatLite; class WeakFieldMap; -#ifndef SWIG // We compute sizes as size_t but cache them as int. This function converts a // computed size to a cached size. Since we don't proceed with serialization // if the total size was > INT_MAX, it is not important what this function @@ -98,22 +116,23 @@ inline int ToIntSize(size_t size) { // // Pay special attention to the initialization state of the object. // 1. The object is "uninitialized" to begin with. -// 2. Call DefaultConstruct() only if the object is uninitialized. -// After the call, the object becomes "initialized". +// 2. Call Construct() or DefaultConstruct() only if the object is +// uninitialized. After the call, the object becomes "initialized". // 3. Call get() and get_mutable() only if the object is initialized. // 4. Call Destruct() only if the object is initialized. // After the call, the object becomes uninitialized. template class ExplicitlyConstructed { public: - void DefaultConstruct() { - new (&union_) T(); - } + void DefaultConstruct() { new (&union_) T(); } - void Destruct() { - get_mutable()->~T(); + template + void Construct(Args&&... args) { + new (&union_) T(std::forward(args)...); } + void Destruct() { get_mutable()->~T(); } + constexpr const T& get() const { return reinterpret_cast(union_); } T* get_mutable() { return reinterpret_cast(&union_); } @@ -128,14 +147,16 @@ class ExplicitlyConstructed { // Default empty string object. Don't use this directly. Instead, call // GetEmptyString() to get the reference. -LIBPROTOBUF_EXPORT extern ExplicitlyConstructed<::std::string> fixed_address_empty_string; +PROTOBUF_EXPORT extern ExplicitlyConstructed + fixed_address_empty_string; -LIBPROTOBUF_EXPORT inline const ::std::string& GetEmptyStringAlreadyInited() { + +PROTOBUF_EXPORT inline const std::string& GetEmptyStringAlreadyInited() { return fixed_address_empty_string.get(); } -LIBPROTOBUF_EXPORT size_t StringSpaceUsedExcludingSelfLong(const string& str); -#endif // SWIG +PROTOBUF_EXPORT size_t StringSpaceUsedExcludingSelfLong(const std::string& str); + } // namespace internal // Interface to light weight protocol messages. @@ -161,15 +182,18 @@ LIBPROTOBUF_EXPORT size_t StringSpaceUsedExcludingSelfLong(const string& str); // is best when you only have a small number of message types linked // into your binary, in which case the size of the protocol buffers // runtime itself is the biggest problem. -class LIBPROTOBUF_EXPORT MessageLite { +// +// Users must not derive from this class. Only the protocol compiler and +// the internal library are allowed to create subclasses. +class PROTOBUF_EXPORT MessageLite { public: inline MessageLite() {} - virtual ~MessageLite() {} + virtual ~MessageLite() = default; // Basic Operations ------------------------------------------------ // Get the name of this message type, e.g. "foo.bar.BazProto". - virtual string GetTypeName() const = 0; + virtual std::string GetTypeName() const = 0; // Construct a new instance of the same type. Ownership is passed to the // caller. @@ -177,14 +201,14 @@ class LIBPROTOBUF_EXPORT MessageLite { // Construct a new instance on the arena. Ownership is passed to the caller // if arena is a NULL. Default implementation for backwards compatibility. - virtual MessageLite* New(::google::protobuf::Arena* arena) const; + virtual MessageLite* New(Arena* arena) const; // Get the arena, if any, associated with this message. Virtual method // required for generic operations but most arena-related operations should - // use the GetArenaNoVirtual() generated-code method. Default implementation + // use the GetArena() generated-code method. Default implementation // to reduce code size by avoiding the need for per-type implementations // when types do not implement arena support. - virtual ::google::protobuf::Arena* GetArena() const { return NULL; } + Arena* GetArena() const { return _internal_metadata_.arena(); } // Get a pointer that may be equal to this message's arena, or may not be. // If the value returned by this method is equal to some arena pointer, then @@ -195,7 +219,9 @@ class LIBPROTOBUF_EXPORT MessageLite { // store the arena pointer directly, and sometimes in a more indirect way, // and allow a fastpath comparison against the arena pointer when it's easy // to obtain. - virtual void* GetMaybeArenaPointer() const { return GetArena(); } + void* GetMaybeArenaPointer() const { + return _internal_metadata_.raw_arena_ptr(); + } // Clear all fields of the message and set them to their default values. // Clear() avoids freeing memory, assuming that any memory allocated @@ -210,12 +236,27 @@ class LIBPROTOBUF_EXPORT MessageLite { // This is not implemented for Lite messages -- it just returns "(cannot // determine missing fields for lite message)". However, it is implemented // for full messages. See message.h. - virtual string InitializationErrorString() const; + virtual std::string InitializationErrorString() const; // If |other| is the exact same class as this, calls MergeFrom(). Otherwise, // results are undefined (probably crash). virtual void CheckTypeAndMergeFrom(const MessageLite& other) = 0; + // These methods return a human-readable summary of the message. Note that + // since the MessageLite interface does not support reflection, there is very + // little information that these methods can provide. They are shadowed by + // methods of the same name on the Message interface which provide much more + // information. The methods here are intended primarily to facilitate code + // reuse for logic that needs to interoperate with both full and lite protos. + // + // The format of the returned string is subject to change, so please do not + // assume it will remain stable over time. + std::string DebugString() const; + std::string ShortDebugString() const { return DebugString(); } + // MessageLite::DebugString is already Utf8 Safe. This is to add compatibility + // with Message. + std::string Utf8DebugString() const { return DebugString(); } + // Parsing --------------------------------------------------------- // Methods for parsing in protocol buffer format. Most of these are // just simple wrappers around MergeFromCodedStream(). Clear() will be @@ -236,9 +277,26 @@ class LIBPROTOBUF_EXPORT MessageLite { // Like ParseFromZeroCopyStream(), but accepts messages that are missing // required fields. bool ParsePartialFromZeroCopyStream(io::ZeroCopyInputStream* input); + // Parse a protocol buffer from a file descriptor. If successful, the entire + // input will be consumed. + bool ParseFromFileDescriptor(int file_descriptor); + // Like ParseFromFileDescriptor(), but accepts messages that are missing + // required fields. + bool ParsePartialFromFileDescriptor(int file_descriptor); + // Parse a protocol buffer from a C++ istream. If successful, the entire + // input will be consumed. + bool ParseFromIstream(std::istream* input); + // Like ParseFromIstream(), but accepts messages that are missing + // required fields. + bool ParsePartialFromIstream(std::istream* input); // Read a protocol buffer from the given zero-copy input stream, expecting // the message to be exactly "size" bytes long. If successful, exactly // this many bytes will have been consumed from the input. + bool MergePartialFromBoundedZeroCopyStream(io::ZeroCopyInputStream* input, + int size); + // Like ParseFromBoundedZeroCopyStream(), but accepts messages that are + // missing required fields. + bool MergeFromBoundedZeroCopyStream(io::ZeroCopyInputStream* input, int size); bool ParseFromBoundedZeroCopyStream(io::ZeroCopyInputStream* input, int size); // Like ParseFromBoundedZeroCopyStream(), but accepts messages that are // missing required fields. @@ -249,10 +307,10 @@ class LIBPROTOBUF_EXPORT MessageLite { // format, matching the encoding output by MessageLite::SerializeToString(). // If you'd like to convert a human-readable string into a protocol buffer // object, see google::protobuf::TextFormat::ParseFromString(). - bool ParseFromString(const string& data); + bool ParseFromString(const std::string& data); // Like ParseFromString(), but accepts messages that are missing // required fields. - bool ParsePartialFromString(const string& data); + bool ParsePartialFromString(const std::string& data); // Parse a protocol buffer contained in an array of bytes. bool ParseFromArray(const void* data, int size); // Like ParseFromArray(), but accepts messages that are missing @@ -269,7 +327,7 @@ class LIBPROTOBUF_EXPORT MessageLite { // (for groups) or input->ConsumedEntireMessage() (for non-groups) after // this returns to verify that the message's end was delimited correctly. // - // ParsefromCodedStream() is implemented as Clear() followed by + // ParseFromCodedStream() is implemented as Clear() followed by // MergeFromCodedStream(). bool MergeFromCodedStream(io::CodedInputStream* input); @@ -278,7 +336,10 @@ class LIBPROTOBUF_EXPORT MessageLite { // // MergeFromCodedStream() is just implemented as MergePartialFromCodedStream() // followed by IsInitialized(). - virtual bool MergePartialFromCodedStream(io::CodedInputStream* input) = 0; + bool MergePartialFromCodedStream(io::CodedInputStream* input); + + // Merge a protocol buffer contained in a string. + bool MergeFromString(const std::string& data); // Serialization --------------------------------------------------- @@ -298,9 +359,9 @@ class LIBPROTOBUF_EXPORT MessageLite { bool SerializePartialToZeroCopyStream(io::ZeroCopyOutputStream* output) const; // Serialize the message and store it in the given string. All required // fields must be set. - bool SerializeToString(string* output) const; + bool SerializeToString(std::string* output) const; // Like SerializeToString(), but allows missing required fields. - bool SerializePartialToString(string* output) const; + bool SerializePartialToString(std::string* output) const; // Serialize the message and store it in the given byte array. All required // fields must be set. bool SerializeToArray(void* data, int size) const; @@ -313,15 +374,27 @@ class LIBPROTOBUF_EXPORT MessageLite { // Note: If you intend to generate many such strings, you may // reduce heap fragmentation by instead re-using the same string // object with calls to SerializeToString(). - string SerializeAsString() const; + std::string SerializeAsString() const; // Like SerializeAsString(), but allows missing required fields. - string SerializePartialAsString() const; - - // Like SerializeToString(), but appends to the data to the string's existing - // contents. All required fields must be set. - bool AppendToString(string* output) const; + std::string SerializePartialAsString() const; + + // Serialize the message and write it to the given file descriptor. All + // required fields must be set. + bool SerializeToFileDescriptor(int file_descriptor) const; + // Like SerializeToFileDescriptor(), but allows missing required fields. + bool SerializePartialToFileDescriptor(int file_descriptor) const; + // Serialize the message and write it to the given C++ ostream. All + // required fields must be set. + bool SerializeToOstream(std::ostream* output) const; + // Like SerializeToOstream(), but allows missing required fields. + bool SerializePartialToOstream(std::ostream* output) const; + + // Like SerializeToString(), but appends to the data to the string's + // existing contents. All required fields must be set. + bool AppendToString(std::string* output) const; // Like AppendToString(), but allows missing required fields. - bool AppendPartialToString(string* output) const; + bool AppendPartialToString(std::string* output) const; + // Computes the serialized size of the message. This recursively calls // ByteSizeLong() on all embedded messages. @@ -331,16 +404,15 @@ class LIBPROTOBUF_EXPORT MessageLite { virtual size_t ByteSizeLong() const = 0; // Legacy ByteSize() API. - PROTOBUF_RUNTIME_DEPRECATED("Please use ByteSizeLong() instead") - int ByteSize() const { - return internal::ToIntSize(ByteSizeLong()); - } + PROTOBUF_DEPRECATED_MSG("Please use ByteSizeLong() instead") + int ByteSize() const { return internal::ToIntSize(ByteSizeLong()); } // Serializes the message without recomputing the size. The message must not // have changed since the last call to ByteSize(), and the value returned by // ByteSize must be non-negative. Otherwise the results are undefined. - virtual void SerializeWithCachedSizes( - io::CodedOutputStream* output) const; + void SerializeWithCachedSizes(io::CodedOutputStream* output) const { + output->SetCur(_InternalSerialize(output->Cur(), output->EpsCopy())); + } // Functions below here are not part of the public interface. It isn't // enforced, but they should be treated as private, and will be private @@ -352,7 +424,7 @@ class LIBPROTOBUF_EXPORT MessageLite { // must point at a byte array of at least ByteSize() bytes. Whether to use // deterministic serialization, e.g., maps in sorted order, is determined by // CodedOutputStream::IsDefaultSerializationDeterministic(). - virtual uint8* SerializeWithCachedSizesToArray(uint8* target) const; + uint8* SerializeWithCachedSizesToArray(uint8* target) const; // Returns the result of the last call to ByteSize(). An embedded message's // size is needed both to serialize it (because embedded messages are @@ -367,30 +439,48 @@ class LIBPROTOBUF_EXPORT MessageLite { // method.) virtual int GetCachedSize() const = 0; - virtual uint8* InternalSerializeWithCachedSizesToArray(bool deterministic, - uint8* target) const; - - protected: - // CastToBase allows generated code to cast a RepeatedPtrField to - // RepeatedPtrFieldBase. We try to restrict access to RepeatedPtrFieldBase - // because it is an implementation detail that user code should not access - // directly. - template - static ::google::protobuf::internal::RepeatedPtrFieldBase* CastToBase( - ::google::protobuf::RepeatedPtrField* repeated) { - return repeated; - } - template - static const ::google::protobuf::internal::RepeatedPtrFieldBase& CastToBase( - const ::google::protobuf::RepeatedPtrField& repeated) { - return repeated; + virtual const char* _InternalParse(const char* /*ptr*/, + internal::ParseContext* /*ctx*/) { + return nullptr; } + protected: template static T* CreateMaybeMessage(Arena* arena) { return Arena::CreateMaybeMessage(arena); } + inline explicit MessageLite(Arena* arena) : _internal_metadata_(arena) {} + + internal::InternalMetadata _internal_metadata_; + + public: + enum ParseFlags { + kMerge = 0, + kParse = 1, + kMergePartial = 2, + kParsePartial = 3, + kMergeWithAliasing = 4, + kParseWithAliasing = 5, + kMergePartialWithAliasing = 6, + kParsePartialWithAliasing = 7 + }; + + template + bool ParseFrom(const T& input); + + // Fast path when conditions match (ie. non-deterministic) + // uint8* _InternalSerialize(uint8* ptr) const; + virtual uint8* _InternalSerialize(uint8* ptr, + io::EpsCopyOutputStream* stream) const = 0; + + // Identical to IsInitialized() except that it logs an error message. + bool IsInitializedWithErrors() const { + if (IsInitialized()) return true; + LogInitializationErrorMessage(); + return false; + } + private: // TODO(gerbens) make this a pure abstract function virtual const void* InternalGetTable() const { return NULL; } @@ -399,26 +489,103 @@ class LIBPROTOBUF_EXPORT MessageLite { friend class Message; friend class internal::WeakFieldMap; + void LogInitializationErrorMessage() const; + + bool MergeFromImpl(io::CodedInputStream* input, ParseFlags parse_flags); + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MessageLite); }; namespace internal { -extern bool LIBPROTOBUF_EXPORT proto3_preserve_unknown_; +template +bool MergeFromImpl(StringPiece input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +extern template bool MergeFromImpl(StringPiece input, + MessageLite* msg, + MessageLite::ParseFlags parse_flags); +extern template bool MergeFromImpl(StringPiece input, + MessageLite* msg, + MessageLite::ParseFlags parse_flags); + +template +bool MergeFromImpl(io::ZeroCopyInputStream* input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +extern template bool MergeFromImpl(io::ZeroCopyInputStream* input, + MessageLite* msg, + MessageLite::ParseFlags parse_flags); +extern template bool MergeFromImpl(io::ZeroCopyInputStream* input, + MessageLite* msg, + MessageLite::ParseFlags parse_flags); + +struct BoundedZCIS { + io::ZeroCopyInputStream* zcis; + int limit; +}; + +template +bool MergeFromImpl(BoundedZCIS input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +extern template bool MergeFromImpl(BoundedZCIS input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +extern template bool MergeFromImpl(BoundedZCIS input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); + +template +struct SourceWrapper; -// DO NOT USE: For migration only. Will be removed when Proto3 defaults to -// preserve unknowns. -inline bool GetProto3PreserveUnknownsDefault() { - return proto3_preserve_unknown_; +template +bool MergeFromImpl(const SourceWrapper& input, MessageLite* msg, + MessageLite::ParseFlags parse_flags) { + return input.template MergeInto(msg, parse_flags); } -// DO NOT USE: For migration only. Will be removed when Proto3 defaults to -// preserve unknowns. -void LIBPROTOBUF_EXPORT SetProto3PreserveUnknownsDefault(bool preserve); } // namespace internal +template +bool MessageLite::ParseFrom(const T& input) { + if (flags & kParse) Clear(); + constexpr bool alias = (flags & kMergeWithAliasing) != 0; + return internal::MergeFromImpl(input, this, flags); +} -} // namespace protobuf +// =================================================================== +// Shutdown support. + + +// Shut down the entire protocol buffers library, deleting all static-duration +// objects allocated by the library or by generated .pb.cc files. +// +// There are two reasons you might want to call this: +// * You use a draconian definition of "memory leak" in which you expect +// every single malloc() to have a corresponding free(), even for objects +// which live until program exit. +// * You are writing a dynamically-loaded library which needs to clean up +// after itself when the library is unloaded. +// +// It is safe to call this multiple times. However, it is not safe to use +// any other part of the protocol buffers library after +// ShutdownProtobufLibrary() has been called. Furthermore this call is not +// thread safe, user needs to synchronize multiple calls. +PROTOBUF_EXPORT void ShutdownProtobufLibrary(); + +namespace internal { + +// Register a function to be called when ShutdownProtocolBuffers() is called. +PROTOBUF_EXPORT void OnShutdown(void (*func)()); +// Run an arbitrary function on an arg +PROTOBUF_EXPORT void OnShutdownRun(void (*f)(const void*), const void* arg); +template +T* OnShutdownDelete(T* p) { + OnShutdownRun([](const void* pp) { delete static_cast(pp); }, p); + return p; +} + +} // namespace internal +} // namespace protobuf } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_MESSAGE_LITE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/metadata_lite.h b/third_party/protobuf-lite/google/protobuf/metadata_lite.h index 454d088c..781a1f54 100644 --- a/third_party/protobuf-lite/google/protobuf/metadata_lite.h +++ b/third_party/protobuf-lite/google/protobuf/metadata_lite.h @@ -31,10 +31,16 @@ #ifndef GOOGLE_PROTOBUF_METADATA_LITE_H__ #define GOOGLE_PROTOBUF_METADATA_LITE_H__ +#include #include #include -#include -#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { @@ -50,48 +56,54 @@ namespace internal { // The tagged pointer uses the LSB to disambiguate cases, and uses bit 0 == 0 to // indicate an arena pointer and bit 0 == 1 to indicate a UFS+Arena-container // pointer. -template -class InternalMetadataWithArenaBase { +class InternalMetadata { public: - InternalMetadataWithArenaBase() : ptr_(NULL) {} - explicit InternalMetadataWithArenaBase(Arena* arena) : ptr_(arena) {} + InternalMetadata() : ptr_(nullptr) {} + explicit InternalMetadata(Arena* arena) : ptr_(arena) {} - ~InternalMetadataWithArenaBase() { + template + void Delete() { + // Note that Delete<> should be called not more than once. if (have_unknown_fields() && arena() == NULL) { - delete PtrValue(); + delete PtrValue>(); } - ptr_ = NULL; } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE const T& unknown_fields() const { - if (GOOGLE_PREDICT_FALSE(have_unknown_fields())) { - return PtrValue()->unknown_fields; + PROTOBUF_ALWAYS_INLINE Arena* arena() const { + if (PROTOBUF_PREDICT_FALSE(have_unknown_fields())) { + return PtrValue()->arena; } else { - return Derived::default_instance(); + return PtrValue(); } } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* mutable_unknown_fields() { - if (GOOGLE_PREDICT_TRUE(have_unknown_fields())) { - return &PtrValue()->unknown_fields; - } else { - return mutable_unknown_fields_slow(); - } + PROTOBUF_ALWAYS_INLINE bool have_unknown_fields() const { + return PtrTag() == kTagContainer; } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE Arena* arena() const { - if (GOOGLE_PREDICT_FALSE(have_unknown_fields())) { - return PtrValue()->arena; + PROTOBUF_ALWAYS_INLINE void* raw_arena_ptr() const { return ptr_; } + + template + PROTOBUF_ALWAYS_INLINE const T& unknown_fields( + const T& (*default_instance)()) const { + if (PROTOBUF_PREDICT_FALSE(have_unknown_fields())) { + return PtrValue>()->unknown_fields; } else { - return PtrValue(); + return default_instance(); } } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE bool have_unknown_fields() const { - return PtrTag() == kTagContainer; + template + PROTOBUF_ALWAYS_INLINE T* mutable_unknown_fields() { + if (PROTOBUF_PREDICT_TRUE(have_unknown_fields())) { + return &PtrValue>()->unknown_fields; + } else { + return mutable_unknown_fields_slow(); + } } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Swap(Derived* other) { + template + PROTOBUF_ALWAYS_INLINE void Swap(InternalMetadata* other) { // Semantics here are that we swap only the unknown fields, not the arena // pointer. We cannot simply swap ptr_ with other->ptr_ because we need to // maintain our own arena ptr. Also, our ptr_ and other's ptr_ may be in @@ -99,26 +111,24 @@ class InternalMetadataWithArenaBase { // cannot simply swap ptr_ and then restore the arena pointers. We reuse // UFS's swap implementation instead. if (have_unknown_fields() || other->have_unknown_fields()) { - static_cast(this)->DoSwap(other->mutable_unknown_fields()); + DoSwap(other->mutable_unknown_fields()); } } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void MergeFrom(const Derived& other) { + template + PROTOBUF_ALWAYS_INLINE void MergeFrom(const InternalMetadata& other) { if (other.have_unknown_fields()) { - static_cast(this)->DoMergeFrom(other.unknown_fields()); + DoMergeFrom(other.unknown_fields(nullptr)); } } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Clear() { + template + PROTOBUF_ALWAYS_INLINE void Clear() { if (have_unknown_fields()) { - static_cast(this)->DoClear(); + DoClear(); } } - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void* raw_arena_ptr() const { - return ptr_; - } - private: void* ptr_; @@ -129,96 +139,110 @@ class InternalMetadataWithArenaBase { // ptr_ is a Container*. kTagContainer = 1, }; - static const intptr_t kPtrTagMask = 1; - static const intptr_t kPtrValueMask = ~kPtrTagMask; + static constexpr intptr_t kPtrTagMask = 1; + static constexpr intptr_t kPtrValueMask = ~kPtrTagMask; // Accessors for pointer tag and pointer value. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE int PtrTag() const { + PROTOBUF_ALWAYS_INLINE int PtrTag() const { return reinterpret_cast(ptr_) & kPtrTagMask; } - template U* PtrValue() const { - return reinterpret_cast( - reinterpret_cast(ptr_) & kPtrValueMask); + template + U* PtrValue() const { + return reinterpret_cast(reinterpret_cast(ptr_) & + kPtrValueMask); } // If ptr_'s tag is kTagContainer, it points to an instance of this struct. - struct Container { - T unknown_fields; + struct ContainerBase { Arena* arena; }; - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE T* mutable_unknown_fields_slow() { + template + struct Container : public ContainerBase { + T unknown_fields; + }; + + template + PROTOBUF_NOINLINE T* mutable_unknown_fields_slow() { Arena* my_arena = arena(); - Container* container = Arena::Create(my_arena); + Container* container = Arena::Create>(my_arena); // Two-step assignment works around a bug in clang's static analyzer: // https://bugs.llvm.org/show_bug.cgi?id=34198. ptr_ = container; - ptr_ = reinterpret_cast( - reinterpret_cast(ptr_) | kTagContainer); + ptr_ = reinterpret_cast(reinterpret_cast(ptr_) | + kTagContainer); container->arena = my_arena; return &(container->unknown_fields); } -}; - -// We store unknown fields as a string right now, because there is currently no -// good interface for reading unknown fields into an ArenaString. We may want -// to revisit this to allow unknown fields to be parsed onto the Arena. -class InternalMetadataWithArenaLite - : public InternalMetadataWithArenaBase { - public: - InternalMetadataWithArenaLite() {} - explicit InternalMetadataWithArenaLite(Arena* arena) - : InternalMetadataWithArenaBase(arena) {} + // Templated functions. - void DoSwap(string* other) { - mutable_unknown_fields()->swap(*other); - } - - void DoMergeFrom(const string& other) { - mutable_unknown_fields()->append(other); + template + void DoClear() { + mutable_unknown_fields()->Clear(); } - void DoClear() { - mutable_unknown_fields()->clear(); + template + void DoMergeFrom(const T& other) { + mutable_unknown_fields()->MergeFrom(other); } - static const string& default_instance() { - return GetEmptyStringAlreadyInited(); + template + void DoSwap(T* other) { + mutable_unknown_fields()->Swap(other); } }; +// String Template specializations. + +template <> +inline void InternalMetadata::DoClear() { + mutable_unknown_fields()->clear(); +} + +template <> +inline void InternalMetadata::DoMergeFrom( + const std::string& other) { + mutable_unknown_fields()->append(other); +} + +template <> +inline void InternalMetadata::DoSwap(std::string* other) { + mutable_unknown_fields()->swap(*other); +} + // This helper RAII class is needed to efficiently parse unknown fields. We // should only call mutable_unknown_fields if there are actual unknown fields. -// The obvious thing to just use a stack string and swap it at the end of the -// parse won't work, because the destructor of StringOutputStream needs to be -// called before we can modify the string (it check-fails). Using +// The obvious thing to just use a stack string and swap it at the end of +// the parse won't work, because the destructor of StringOutputStream needs to +// be called before we can modify the string (it check-fails). Using // LiteUnknownFieldSetter setter(&_internal_metadata_); // StringOutputStream stream(setter.buffer()); // guarantees that the string is only swapped after stream is destroyed. -class LIBPROTOBUF_EXPORT LiteUnknownFieldSetter { +class PROTOBUF_EXPORT LiteUnknownFieldSetter { public: - explicit LiteUnknownFieldSetter(InternalMetadataWithArenaLite* metadata) + explicit LiteUnknownFieldSetter(InternalMetadata* metadata) : metadata_(metadata) { if (metadata->have_unknown_fields()) { - buffer_.swap(*metadata->mutable_unknown_fields()); + buffer_.swap(*metadata->mutable_unknown_fields()); } } ~LiteUnknownFieldSetter() { - if (!buffer_.empty()) metadata_->mutable_unknown_fields()->swap(buffer_); + if (!buffer_.empty()) + metadata_->mutable_unknown_fields()->swap(buffer_); } - string* buffer() { return &buffer_; } + std::string* buffer() { return &buffer_; } private: - InternalMetadataWithArenaLite* metadata_; - string buffer_; + InternalMetadata* metadata_; + std::string buffer_; }; } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_METADATA_LITE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/parse_context.h b/third_party/protobuf-lite/google/protobuf/parse_context.h new file mode 100644 index 00000000..c0427bc5 --- /dev/null +++ b/third_party/protobuf-lite/google/protobuf/parse_context.h @@ -0,0 +1,809 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PARSE_CONTEXT_H__ +#define GOOGLE_PROTOBUF_PARSE_CONTEXT_H__ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace google { +namespace protobuf { + +class UnknownFieldSet; +class DescriptorPool; +class MessageFactory; + +namespace internal { + +// Template code below needs to know about the existence of these functions. +PROTOBUF_EXPORT void WriteVarint(uint32 num, uint64 val, std::string* s); +PROTOBUF_EXPORT void WriteLengthDelimited(uint32 num, StringPiece val, + std::string* s); +// Inline because it is just forwarding to s->WriteVarint +inline void WriteVarint(uint32 num, uint64 val, UnknownFieldSet* s); +inline void WriteLengthDelimited(uint32 num, StringPiece val, + UnknownFieldSet* s); + + +// The basic abstraction the parser is designed for is a slight modification +// of the ZeroCopyInputStream (ZCIS) abstraction. A ZCIS presents a serialized +// stream as a series of buffers that concatenate to the full stream. +// Pictorially a ZCIS presents a stream in chunks like so +// [---------------------------------------------------------------] +// [---------------------] chunk 1 +// [----------------------------] chunk 2 +// chunk 3 [--------------] +// +// Where the '-' represent the bytes which are vertically lined up with the +// bytes of the stream. The proto parser requires its input to be presented +// similarly with the extra +// property that each chunk has kSlopBytes past its end that overlaps with the +// first kSlopBytes of the next chunk, or if there is no next chunk at least its +// still valid to read those bytes. Again, pictorially, we now have +// +// [---------------------------------------------------------------] +// [-------------------....] chunk 1 +// [------------------------....] chunk 2 +// chunk 3 [------------------..**] +// chunk 4 [--****] +// Here '-' mean the bytes of the stream or chunk and '.' means bytes past the +// chunk that match up with the start of the next chunk. Above each chunk has +// 4 '.' after the chunk. In the case these 'overflow' bytes represents bytes +// past the stream, indicated by '*' above, their values are unspecified. It is +// still legal to read them (ie. should not segfault). Reading past the +// end should be detected by the user and indicated as an error. +// +// The reason for this, admittedly, unconventional invariant is to ruthlessly +// optimize the protobuf parser. Having an overlap helps in two important ways. +// Firstly it alleviates having to performing bounds checks if a piece of code +// is guaranteed to not read more than kSlopBytes. Secondly, and more +// importantly, the protobuf wireformat is such that reading a key/value pair is +// always less than 16 bytes. This removes the need to change to next buffer in +// the middle of reading primitive values. Hence there is no need to store and +// load the current position. + +class PROTOBUF_EXPORT EpsCopyInputStream { + public: + enum { kSlopBytes = 16, kMaxCordBytesToCopy = 512 }; + + explicit EpsCopyInputStream(bool enable_aliasing) + : aliasing_(enable_aliasing ? kOnPatch : kNoAliasing) {} + + void BackUp(const char* ptr) { + GOOGLE_DCHECK(ptr <= buffer_end_ + kSlopBytes); + int count; + if (next_chunk_ == buffer_) { + count = static_cast(buffer_end_ + kSlopBytes - ptr); + } else { + count = size_ + static_cast(buffer_end_ - ptr); + } + if (count > 0) StreamBackUp(count); + } + + // If return value is negative it's an error + PROTOBUF_MUST_USE_RESULT int PushLimit(const char* ptr, int limit) { + GOOGLE_DCHECK(limit >= 0 && limit <= INT_MAX - kSlopBytes); + // This add is safe due to the invariant above, because + // ptr - buffer_end_ <= kSlopBytes. + limit += static_cast(ptr - buffer_end_); + limit_end_ = buffer_end_ + (std::min)(0, limit); + auto old_limit = limit_; + limit_ = limit; + return old_limit - limit; + } + + PROTOBUF_MUST_USE_RESULT bool PopLimit(int delta) { + if (PROTOBUF_PREDICT_FALSE(!EndedAtLimit())) return false; + limit_ = limit_ + delta; + // TODO(gerbens) We could remove this line and hoist the code to + // DoneFallback. Study the perf/bin-size effects. + limit_end_ = buffer_end_ + (std::min)(0, limit_); + return true; + } + + PROTOBUF_MUST_USE_RESULT const char* Skip(const char* ptr, int size) { + if (size <= buffer_end_ + kSlopBytes - ptr) { + return ptr + size; + } + return SkipFallback(ptr, size); + } + PROTOBUF_MUST_USE_RESULT const char* ReadString(const char* ptr, int size, + std::string* s) { + if (size <= buffer_end_ + kSlopBytes - ptr) { + s->assign(ptr, size); + return ptr + size; + } + return ReadStringFallback(ptr, size, s); + } + PROTOBUF_MUST_USE_RESULT const char* AppendString(const char* ptr, int size, + std::string* s) { + if (size <= buffer_end_ + kSlopBytes - ptr) { + s->append(ptr, size); + return ptr + size; + } + return AppendStringFallback(ptr, size, s); + } + + template + PROTOBUF_MUST_USE_RESULT const char* ReadRepeatedFixed(const char* ptr, + Tag expected_tag, + RepeatedField* out); + + template + PROTOBUF_MUST_USE_RESULT const char* ReadPackedFixed(const char* ptr, + int size, + RepeatedField* out); + template + PROTOBUF_MUST_USE_RESULT const char* ReadPackedVarint(const char* ptr, + Add add); + + uint32 LastTag() const { return last_tag_minus_1_ + 1; } + bool ConsumeEndGroup(uint32 start_tag) { + bool res = last_tag_minus_1_ == start_tag; + last_tag_minus_1_ = 0; + return res; + } + bool EndedAtLimit() const { return last_tag_minus_1_ == 0; } + bool EndedAtEndOfStream() const { return last_tag_minus_1_ == 1; } + void SetLastTag(uint32 tag) { last_tag_minus_1_ = tag - 1; } + void SetEndOfStream() { last_tag_minus_1_ = 1; } + bool IsExceedingLimit(const char* ptr) { + return ptr > limit_end_ && + (next_chunk_ == nullptr || ptr - buffer_end_ > limit_); + } + int BytesUntilLimit(const char* ptr) const { + return limit_ + static_cast(buffer_end_ - ptr); + } + // Returns true if more data is available, if false is returned one has to + // call Done for further checks. + bool DataAvailable(const char* ptr) { return ptr < limit_end_; } + + protected: + // Returns true is limit (either an explicit limit or end of stream) is + // reached. It aligns *ptr across buffer seams. + // If limit is exceeded it returns true and ptr is set to null. + bool DoneWithCheck(const char** ptr, int d) { + GOOGLE_DCHECK(*ptr); + if (PROTOBUF_PREDICT_TRUE(*ptr < limit_end_)) return false; + // No need to fetch buffer if we ended on a limit in the slop region + if ((*ptr - buffer_end_) == limit_) return true; + auto res = DoneFallback(*ptr, d); + *ptr = res.first; + return res.second; + } + + const char* InitFrom(StringPiece flat) { + overall_limit_ = 0; + if (flat.size() > kSlopBytes) { + limit_ = kSlopBytes; + limit_end_ = buffer_end_ = flat.data() + flat.size() - kSlopBytes; + next_chunk_ = buffer_; + if (aliasing_ == kOnPatch) aliasing_ = kNoDelta; + return flat.data(); + } else { + std::memcpy(buffer_, flat.data(), flat.size()); + limit_ = 0; + limit_end_ = buffer_end_ = buffer_ + flat.size(); + next_chunk_ = nullptr; + if (aliasing_ == kOnPatch) { + aliasing_ = reinterpret_cast(flat.data()) - + reinterpret_cast(buffer_); + } + return buffer_; + } + } + + const char* InitFrom(io::ZeroCopyInputStream* zcis); + + const char* InitFrom(io::ZeroCopyInputStream* zcis, int limit) { + if (limit == -1) return InitFrom(zcis); + overall_limit_ = limit; + auto res = InitFrom(zcis); + limit_ = limit - static_cast(buffer_end_ - res); + limit_end_ = buffer_end_ + (std::min)(0, limit_); + return res; + } + + private: + const char* limit_end_; // buffer_end_ + min(limit_, 0) + const char* buffer_end_; + const char* next_chunk_; + int size_; + int limit_; // relative to buffer_end_; + io::ZeroCopyInputStream* zcis_ = nullptr; + char buffer_[2 * kSlopBytes] = {}; + enum { kNoAliasing = 0, kOnPatch = 1, kNoDelta = 2 }; + std::uintptr_t aliasing_ = kNoAliasing; + // This variable is used to communicate how the parse ended, in order to + // completely verify the parsed data. A wire-format parse can end because of + // one of the following conditions: + // 1) A parse can end on a pushed limit. + // 2) A parse can end on End Of Stream (EOS). + // 3) A parse can end on 0 tag (only valid for toplevel message). + // 4) A parse can end on an end-group tag. + // This variable should always be set to 0, which indicates case 1. If the + // parse terminated due to EOS (case 2), it's set to 1. In case the parse + // ended due to a terminating tag (case 3 and 4) it's set to (tag - 1). + // This var doesn't really belong in EpsCopyInputStream and should be part of + // the ParseContext, but case 2 is most easily and optimally implemented in + // DoneFallback. + uint32 last_tag_minus_1_ = 0; + int overall_limit_ = INT_MAX; // Overall limit independent of pushed limits. + // Pretty random large number that seems like a safe allocation on most + // systems. TODO(gerbens) do we need to set this as build flag? + enum { kSafeStringSize = 50000000 }; + + std::pair DoneFallback(const char* ptr, int d); + const char* Next(int overrun, int d); + const char* SkipFallback(const char* ptr, int size); + const char* AppendStringFallback(const char* ptr, int size, std::string* str); + const char* ReadStringFallback(const char* ptr, int size, std::string* str); + bool StreamNext(const void** data) { + bool res = zcis_->Next(data, &size_); + if (res) overall_limit_ -= size_; + return res; + } + void StreamBackUp(int count) { + zcis_->BackUp(count); + overall_limit_ += count; + } + + template + const char* AppendSize(const char* ptr, int size, const A& append) { + int chunk_size = buffer_end_ + kSlopBytes - ptr; + do { + GOOGLE_DCHECK(size > chunk_size); + append(ptr, chunk_size); + ptr += chunk_size; + size -= chunk_size; + // DoneFallBack asserts it isn't called when exactly on the limit. If this + // happens we fail the parse, as we are at the limit and still more bytes + // to read. + if (limit_ == kSlopBytes) return nullptr; + auto res = DoneFallback(ptr, -1); + if (res.second) return nullptr; // If done we passed the limit + ptr = res.first; + chunk_size = buffer_end_ + kSlopBytes - ptr; + } while (size > chunk_size); + append(ptr, size); + return ptr + size; + } + + // AppendUntilEnd appends data until a limit (either a PushLimit or end of + // stream. Normal payloads are from length delimited fields which have an + // explicit size. Reading until limit only comes when the string takes + // the place of a protobuf, ie RawMessage/StringRawMessage, lazy fields and + // implicit weak messages. We keep these methods private and friend them. + template + const char* AppendUntilEnd(const char* ptr, const A& append) { + while (!DoneWithCheck(&ptr, -1)) { + append(ptr, limit_end_ - ptr); + ptr = limit_end_; + } + return ptr; + } + + PROTOBUF_MUST_USE_RESULT const char* AppendString(const char* ptr, + std::string* str) { + return AppendUntilEnd( + ptr, [str](const char* p, ptrdiff_t s) { str->append(p, s); }); + } + friend class ImplicitWeakMessage; +}; + +// ParseContext holds all data that is global to the entire parse. Most +// importantly it contains the input stream, but also recursion depth and also +// stores the end group tag, in case a parser ended on a endgroup, to verify +// matching start/end group tags. +class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { + public: + struct Data { + const DescriptorPool* pool = nullptr; + MessageFactory* factory = nullptr; + }; + + template + ParseContext(int depth, bool aliasing, const char** start, T&&... args) + : EpsCopyInputStream(aliasing), depth_(depth) { + *start = InitFrom(std::forward(args)...); + } + + void TrackCorrectEnding() { group_depth_ = 0; } + + bool Done(const char** ptr) { return DoneWithCheck(ptr, group_depth_); } + bool DoneNoSlopCheck(const char** ptr) { return DoneWithCheck(ptr, -1); } + + int depth() const { return depth_; } + + Data& data() { return data_; } + const Data& data() const { return data_; } + + template + PROTOBUF_MUST_USE_RESULT const char* ParseMessage(T* msg, const char* ptr); + // We outline when the type is generic and we go through a virtual + const char* ParseMessage(MessageLite* msg, const char* ptr); + const char* ParseMessage(Message* msg, const char* ptr); + + template + PROTOBUF_MUST_USE_RESULT PROTOBUF_ALWAYS_INLINE const char* ParseGroup( + T* msg, const char* ptr, uint32 tag) { + if (--depth_ < 0) return nullptr; + group_depth_++; + ptr = msg->_InternalParse(ptr, this); + group_depth_--; + depth_++; + if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(tag))) return nullptr; + return ptr; + } + + private: + // The context keeps an internal stack to keep track of the recursive + // part of the parse state. + // Current depth of the active parser, depth counts down. + // This is used to limit recursion depth (to prevent overflow on malicious + // data), but is also used to index in stack_ to store the current state. + int depth_; + // Unfortunately necessary for the fringe case of ending on 0 or end-group tag + // in the last kSlopBytes of a ZeroCopyInputStream chunk. + int group_depth_ = INT_MIN; + Data data_; +}; + +template +bool ExpectTag(const char* ptr) { + if (tag < 128) { + return *ptr == tag; + } else { + static_assert(tag < 128 * 128, "We only expect tags for 1 or 2 bytes"); + char buf[2] = {static_cast(tag | 0x80), static_cast(tag >> 7)}; + return std::memcmp(ptr, buf, 2) == 0; + } +} + +template +struct EndianHelper; + +template <> +struct EndianHelper<1> { + static uint8 Load(const void* p) { return *static_cast(p); } +}; + +template <> +struct EndianHelper<2> { + static uint16 Load(const void* p) { + uint16 tmp; + std::memcpy(&tmp, p, 2); +#ifndef PROTOBUF_LITTLE_ENDIAN + tmp = bswap_16(tmp); +#endif + return tmp; + } +}; + +template <> +struct EndianHelper<4> { + static uint32 Load(const void* p) { + uint32 tmp; + std::memcpy(&tmp, p, 4); +#ifndef PROTOBUF_LITTLE_ENDIAN + tmp = bswap_32(tmp); +#endif + return tmp; + } +}; + +template <> +struct EndianHelper<8> { + static uint64 Load(const void* p) { + uint64 tmp; + std::memcpy(&tmp, p, 8); +#ifndef PROTOBUF_LITTLE_ENDIAN + tmp = bswap_64(tmp); +#endif + return tmp; + } +}; + +template +T UnalignedLoad(const char* p) { + auto tmp = EndianHelper::Load(p); + T res; + memcpy(&res, &tmp, sizeof(T)); + return res; +} + +PROTOBUF_EXPORT +std::pair VarintParseSlow32(const char* p, uint32 res); +PROTOBUF_EXPORT +std::pair VarintParseSlow64(const char* p, uint32 res); + +inline const char* VarintParseSlow(const char* p, uint32 res, uint32* out) { + auto tmp = VarintParseSlow32(p, res); + *out = tmp.second; + return tmp.first; +} + +inline const char* VarintParseSlow(const char* p, uint32 res, uint64* out) { + auto tmp = VarintParseSlow64(p, res); + *out = tmp.second; + return tmp.first; +} + +template +PROTOBUF_MUST_USE_RESULT const char* VarintParse(const char* p, T* out) { + auto ptr = reinterpret_cast(p); + uint32 res = ptr[0]; + if (!(res & 0x80)) { + *out = res; + return p + 1; + } + uint32 byte = ptr[1]; + res += (byte - 1) << 7; + if (!(byte & 0x80)) { + *out = res; + return p + 2; + } + return VarintParseSlow(p, res, out); +} + +// Used for tags, could read up to 5 bytes which must be available. +// Caller must ensure its safe to call. + +PROTOBUF_EXPORT +std::pair ReadTagFallback(const char* p, uint32 res); + +// Same as ParseVarint but only accept 5 bytes at most. +inline const char* ReadTag(const char* p, uint32* out, uint32 /*max_tag*/ = 0) { + uint32 res = static_cast(p[0]); + if (res < 128) { + *out = res; + return p + 1; + } + uint32 second = static_cast(p[1]); + res += (second - 1) << 7; + if (second < 128) { + *out = res; + return p + 2; + } + auto tmp = ReadTagFallback(p, res); + *out = tmp.second; + return tmp.first; +} + +// Decode 2 consecutive bytes of a varint and returns the value, shifted left +// by 1. It simultaneous updates *ptr to *ptr + 1 or *ptr + 2 depending if the +// first byte's continuation bit is set. +// If bit 15 of return value is set (equivalent to the continuation bits of both +// bytes being set) the varint continues, otherwise the parse is done. On x86 +// movsx eax, dil +// add edi, eax +// adc [rsi], 1 +// add eax, eax +// and eax, edi +inline uint32 DecodeTwoBytes(const char** ptr) { + uint32 value = UnalignedLoad(*ptr); + // Sign extend the low byte continuation bit + uint32_t x = static_cast(value); + // This add is an amazing operation, it cancels the low byte continuation bit + // from y transferring it to the carry. Simultaneously it also shifts the 7 + // LSB left by one tightly against high byte varint bits. Hence value now + // contains the unpacked value shifted left by 1. + value += x; + // Use the carry to update the ptr appropriately. + *ptr += value < x ? 2 : 1; + return value & (x + x); // Mask out the high byte iff no continuation +} + +// More efficient varint parsing for big varints +inline const char* ParseBigVarint(const char* p, uint64* out) { + auto pnew = p; + auto tmp = DecodeTwoBytes(&pnew); + uint64 res = tmp >> 1; + if (PROTOBUF_PREDICT_TRUE(std::int16_t(tmp) >= 0)) { + *out = res; + return pnew; + } + for (std::uint32_t i = 1; i < 5; i++) { + pnew = p + 2 * i; + tmp = DecodeTwoBytes(&pnew); + res += (static_cast(tmp) - 2) << (14 * i - 1); + if (PROTOBUF_PREDICT_TRUE(std::int16_t(tmp) >= 0)) { + *out = res; + return pnew; + } + } + return nullptr; +} + +PROTOBUF_EXPORT +std::pair ReadSizeFallback(const char* p, uint32 first); +// Used for tags, could read up to 5 bytes which must be available. Additionally +// it makes sure the unsigned value fits a int32, otherwise returns nullptr. +// Caller must ensure its safe to call. +inline uint32 ReadSize(const char** pp) { + auto p = *pp; + uint32 res = static_cast(p[0]); + if (res < 128) { + *pp = p + 1; + return res; + } + auto x = ReadSizeFallback(p, res); + *pp = x.first; + return x.second; +} + +// Some convenience functions to simplify the generated parse loop code. +// Returning the value and updating the buffer pointer allows for nicer +// function composition. We rely on the compiler to inline this. +// Also in debug compiles having local scoped variables tend to generated +// stack frames that scale as O(num fields). +inline uint64 ReadVarint64(const char** p) { + uint64 tmp; + *p = VarintParse(*p, &tmp); + return tmp; +} + +inline uint32 ReadVarint32(const char** p) { + uint32 tmp; + *p = VarintParse(*p, &tmp); + return tmp; +} + +inline int64 ReadVarintZigZag64(const char** p) { + uint64 tmp; + *p = VarintParse(*p, &tmp); + return WireFormatLite::ZigZagDecode64(tmp); +} + +inline int32 ReadVarintZigZag32(const char** p) { + uint64 tmp; + *p = VarintParse(*p, &tmp); + return WireFormatLite::ZigZagDecode32(static_cast(tmp)); +} + +template +PROTOBUF_MUST_USE_RESULT const char* ParseContext::ParseMessage( + T* msg, const char* ptr) { + int size = ReadSize(&ptr); + if (!ptr) return nullptr; + auto old = PushLimit(ptr, size); + if (--depth_ < 0) return nullptr; + ptr = msg->_InternalParse(ptr, this); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + depth_++; + if (!PopLimit(old)) return nullptr; + return ptr; +} + +template +const char* EpsCopyInputStream::ReadPackedVarint(const char* ptr, Add add) { + int size = ReadSize(&ptr); + if (ptr == nullptr) return nullptr; + auto old = PushLimit(ptr, size); + if (old < 0) return nullptr; + while (!DoneWithCheck(&ptr, -1)) { + uint64 varint; + ptr = VarintParse(ptr, &varint); + if (!ptr) return nullptr; + add(varint); + } + if (!PopLimit(old)) return nullptr; + return ptr; +} + +// Helper for verification of utf8 +PROTOBUF_EXPORT +bool VerifyUTF8(StringPiece s, const char* field_name); + +inline bool VerifyUTF8(const std::string* s, const char* field_name) { + return VerifyUTF8(*s, field_name); +} + +// All the string parsers with or without UTF checking and for all CTypes. +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParser( + std::string* s, const char* ptr, ParseContext* ctx); + + +// Add any of the following lines to debug which parse function is failing. + +#define GOOGLE_PROTOBUF_ASSERT_RETURN(predicate, ret) \ + if (!(predicate)) { \ + /* ::raise(SIGINT); */ \ + /* GOOGLE_LOG(ERROR) << "Parse failure"; */ \ + return ret; \ + } + +#define GOOGLE_PROTOBUF_PARSER_ASSERT(predicate) \ + GOOGLE_PROTOBUF_ASSERT_RETURN(predicate, nullptr) + +template +PROTOBUF_MUST_USE_RESULT const char* FieldParser(uint64 tag, T& field_parser, + const char* ptr, + ParseContext* ctx) { + uint32 number = tag >> 3; + GOOGLE_PROTOBUF_PARSER_ASSERT(number != 0); + using WireType = internal::WireFormatLite::WireType; + switch (tag & 7) { + case WireType::WIRETYPE_VARINT: { + uint64 value; + ptr = VarintParse(ptr, &value); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + field_parser.AddVarint(number, value); + break; + } + case WireType::WIRETYPE_FIXED64: { + uint64 value = UnalignedLoad(ptr); + ptr += 8; + field_parser.AddFixed64(number, value); + break; + } + case WireType::WIRETYPE_LENGTH_DELIMITED: { + ptr = field_parser.ParseLengthDelimited(number, ptr, ctx); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + break; + } + case WireType::WIRETYPE_START_GROUP: { + ptr = field_parser.ParseGroup(number, ptr, ctx); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + break; + } + case WireType::WIRETYPE_END_GROUP: { + GOOGLE_LOG(FATAL) << "Can't happen"; + break; + } + case WireType::WIRETYPE_FIXED32: { + uint32 value = UnalignedLoad(ptr); + ptr += 4; + field_parser.AddFixed32(number, value); + break; + } + default: + return nullptr; + } + return ptr; +} + +template +PROTOBUF_MUST_USE_RESULT const char* WireFormatParser(T& field_parser, + const char* ptr, + ParseContext* ctx) { + while (!ctx->Done(&ptr)) { + uint32 tag; + ptr = ReadTag(ptr, &tag); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr); + if (tag == 0 || (tag & 7) == 4) { + ctx->SetLastTag(tag); + return ptr; + } + ptr = FieldParser(tag, field_parser, ptr, ctx); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr); + } + return ptr; +} + +// The packed parsers parse repeated numeric primitives directly into the +// corresponding field + +// These are packed varints +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedInt32Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedUInt32Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedInt64Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedUInt64Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedSInt32Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedSInt64Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedEnumParser( + void* object, const char* ptr, ParseContext* ctx); + +template +PROTOBUF_EXPORT_TEMPLATE_DEFINE +PROTOBUF_MUST_USE_RESULT const + char* PackedEnumParser(void* object, const char* ptr, ParseContext* ctx, + bool (*is_valid)(int), InternalMetadata* metadata, + int field_num) { + return ctx->ReadPackedVarint( + ptr, [object, is_valid, metadata, field_num](uint64 val) { + if (is_valid(val)) { + static_cast*>(object)->Add(val); + } else { + WriteVarint(field_num, val, metadata->mutable_unknown_fields()); + } + }); +} + +template +PROTOBUF_EXPORT_TEMPLATE_DEFINE +PROTOBUF_MUST_USE_RESULT const + char* PackedEnumParserArg(void* object, const char* ptr, ParseContext* ctx, + bool (*is_valid)(const void*, int), + const void* data, InternalMetadata* metadata, + int field_num) { + return ctx->ReadPackedVarint( + ptr, [object, is_valid, data, metadata, field_num](uint64 val) { + if (is_valid(data, val)) { + static_cast*>(object)->Add(val); + } else { + WriteVarint(field_num, val, metadata->mutable_unknown_fields()); + } + }); +} + +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedBoolParser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedFixed32Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedSFixed32Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedFixed64Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedSFixed64Parser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedFloatParser( + void* object, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* PackedDoubleParser( + void* object, const char* ptr, ParseContext* ctx); + +// This is the only recursive parser. +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownGroupLiteParse( + std::string* unknown, const char* ptr, ParseContext* ctx); +// This is a helper to for the UnknownGroupLiteParse but is actually also +// useful in the generated code. It uses overload on std::string* vs +// UnknownFieldSet* to make the generated code isomorphic between full and lite. +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownFieldParse( + uint32 tag, std::string* unknown, const char* ptr, ParseContext* ctx); + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include + +#endif // GOOGLE_PROTOBUF_PARSE_CONTEXT_H__ diff --git a/third_party/protobuf-lite/arenastring.cc b/third_party/protobuf-lite/google/protobuf/port.h similarity index 81% rename from third_party/protobuf-lite/arenastring.cc rename to third_party/protobuf-lite/google/protobuf/port.h index 7f33a0c8..555fd4eb 100644 --- a/third_party/protobuf-lite/arenastring.cc +++ b/third_party/protobuf-lite/google/protobuf/port.h @@ -28,16 +28,16 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// The ArenaString implementation is not included in the open-source release. Do -// not include this file in the distribution. +// A common header that is included across all protobuf headers. We do our best +// to avoid #defining any macros here; instead we generally put macros in +// port_def.inc and port_undef.inc so they are not visible from outside of +// protobuf. -#include +#ifndef GOOGLE_PROTOBUF_PORT_H__ +#define GOOGLE_PROTOBUF_PORT_H__ -namespace google { -namespace protobuf { -namespace internal { +#include -} // namespace internal -} // namespace protobuf -} // namespace google + +#endif // GOOGLE_PROTOBUF_PORT_H__ diff --git a/third_party/protobuf-lite/google/protobuf/port_def.inc b/third_party/protobuf-lite/google/protobuf/port_def.inc new file mode 100644 index 00000000..c524915c --- /dev/null +++ b/third_party/protobuf-lite/google/protobuf/port_def.inc @@ -0,0 +1,486 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file defines common macros that are used in protobuf. +// +// To hide these definitions from the outside world (and to prevent collisions +// if more than one version of protobuf is #included in the same project) you +// must follow this pattern when #including port_def.inc in a header file: +// +// #include "other_header.h" +// #include "message.h" +// // etc. +// +// #include "port_def.inc" // MUST be last header included +// +// // Definitions for this header. +// +// #include "port_undef.inc" +// +// This is a textual header with no include guard, because we want to +// detect/prohibit anytime it is #included twice without a corresponding +// #undef. + +// These macros are private and should always be +// ::util::RetrieveErrorSpace(*this) headers. If any of these errors fire, you +// should either properly #include port_undef.h at the end of your header that +// #includes port.h, or don't #include port.h twice in a .cc file. +#ifdef PROTOBUF_NAMESPACE +#error PROTOBUF_NAMESPACE was previously defined +#endif +#ifdef PROTOBUF_NAMESPACE_ID +#error PROTOBUF_NAMESPACE_ID was previously defined +#endif +#ifdef PROTOBUF_ALWAYS_INLINE +#error PROTOBUF_ALWAYS_INLINE was previously defined +#endif +#ifdef PROTOBUF_COLD +#error PROTOBUF_COLD was previously defined +#endif +#ifdef PROTOBUF_NOINLINE +#error PROTOBUF_NOINLINE was previously defined +#endif +#ifdef PROTOBUF_SECTION_VARIABLE +#error PROTOBUF_SECTION_VARIABLE was previously defined +#endif +#ifdef PROTOBUF_DEPRECATED +#error PROTOBUF_DEPRECATED was previously defined +#endif +#ifdef PROTOBUF_DEPRECATED_MSG +#error PROTOBUF_DEPRECATED_MSG was previously defined +#endif +#ifdef PROTOBUF_FUNC_ALIGN +#error PROTOBUF_FUNC_ALIGN was previously defined +#endif +#ifdef PROTOBUF_RETURNS_NONNULL +#error PROTOBUF_RETURNS_NONNULL was previously defined +#endif +#ifdef PROTOBUF_ATTRIBUTE_REINITIALIZES +#error PROTOBUF_ATTRIBUTE_REINITIALIZES was previously defined +#endif +#ifdef PROTOBUF_RTTI +#error PROTOBUF_RTTI was previously defined +#endif +#ifdef PROTOBUF_VERSION +#error PROTOBUF_VERSION was previously defined +#endif +#ifdef PROTOBUF_VERSION_SUFFIX +#error PROTOBUF_VERSION_SUFFIX was previously defined +#endif +#ifdef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC +#error PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC was previously defined +#endif +#ifdef PROTOBUF_MIN_PROTOC_VERSION +#error PROTOBUF_MIN_PROTOC_VERSION was previously defined +#endif +#ifdef PROTOBUF_PREDICT_TRUE +#error PROTOBUF_PREDICT_TRUE was previously defined +#endif +#ifdef PROTOBUF_PREDICT_FALSE +#error PROTOBUF_PREDICT_FALSE was previously defined +#endif +#ifdef PROTOBUF_FIELD_OFFSET +#error PROTOBUF_FIELD_OFFSET was previously defined +#endif +#ifdef PROTOBUF_LL_FORMAT +#error PROTOBUF_LL_FORMAT was previously defined +#endif +#ifdef PROTOBUF_GUARDED_BY +#error PROTOBUF_GUARDED_BY was previously defined +#endif +#ifdef PROTOBUF_LONGLONG +#error PROTOBUF_LONGLONG was previously defined +#endif +#ifdef PROTOBUF_ULONGLONG +#error PROTOBUF_ULONGLONG was previously defined +#endif +#ifdef PROTOBUF_FALLTHROUGH_INTENDED +#error PROTOBUF_FALLTHROUGH_INTENDED was previously defined +#endif +#ifdef PROTOBUF_EXPORT +#error PROTOBUF_EXPORT was previously defined +#endif +#ifdef PROTOC_EXPORT +#error PROTOC_EXPORT was previously defined +#endif +#ifdef PROTOBUF_MUST_USE_RESULT +#error PROTOBUF_MUST_USE_RESULT was previously defined +#endif +#ifdef PROTOBUF_UNUSED +#error PROTOBUF_UNUSED was previously defined +#endif +#ifdef PROTOBUF_FINAL +#error PROTOBUF_FINAL was previously defined +#endif + + +#define PROTOBUF_NAMESPACE "google::protobuf" +#define PROTOBUF_NAMESPACE_ID google::protobuf +#define PROTOBUF_NAMESPACE_OPEN \ + namespace google { \ + namespace protobuf { +#define PROTOBUF_NAMESPACE_CLOSE \ + } /* namespace protobuf */ \ + } /* namespace google */ + +#if defined(__GNUC__) || defined(__clang__) +#define PROTOBUF_DEPRECATED __attribute__((deprecated)) +#define PROTOBUF_DEPRECATED_ENUM __attribute__((deprecated)) +#define PROTOBUF_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) +#elif defined(_MSC_VER) +#define PROTOBUF_DEPRECATED __declspec(deprecated) +#define PROTOBUF_DEPRECATED_ENUM +#define PROTOBUF_DEPRECATED_MSG(msg) __declspec(deprecated(msg)) +#endif + +#define PROTOBUF_SECTION_VARIABLE(x) +#define PROTOBUF_MUST_USE_RESULT + +// ---------------------------------------------------------------------------- +// Annotations: Some parts of the code have been annotated in ways that might +// be useful to some compilers or tools, but are not supported universally. +// You can #define these annotations yourself if the default implementation +// is not right for you. + +#ifdef GOOGLE_ATTRIBUTE_ALWAYS_INLINE +#define PROTOBUF_ALWAYS_INLINE GOOGLE_ATTRIBUTE_ALWAYS_INLINE +#else +#if defined(__GNUC__) && \ + (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)) +// For functions we want to force inline. +// Introduced in gcc 3.1. +#define PROTOBUF_ALWAYS_INLINE __attribute__((always_inline)) +#else +// Other compilers will have to figure it out for themselves. +#define PROTOBUF_ALWAYS_INLINE +#endif +#endif + +#ifdef GOOGLE_ATTRIBUTE_NOINLINE +#define PROTOBUF_NOINLINE GOOGLE_ATTRIBUTE_NOINLINE +#else +#if defined(__GNUC__) && \ + (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)) +// For functions we want to force not inline. +// Introduced in gcc 3.1. +#define PROTOBUF_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) +// Seems to have been around since at least Visual Studio 2005 +#define PROTOBUF_NOINLINE __declspec(noinline) +#else +// Other compilers will have to figure it out for themselves. +#define PROTOBUF_NOINLINE +#endif +#endif + +#ifdef GOOGLE_ATTRIBUTE_FUNC_ALIGN +#define PROTOBUF_FUNC_ALIGN GOOGLE_ATTRIBUTE_FUNC_ALIGN +#else +#if defined(__clang__) || \ + defined(__GNUC__) && \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +// Function alignment attribute introduced in gcc 4.3 +#define PROTOBUF_FUNC_ALIGN(bytes) __attribute__((aligned(bytes))) +#else +#define PROTOBUF_FUNC_ALIGN(bytes) +#endif +#endif + +#ifdef GOOGLE_PREDICT_TRUE +#define PROTOBUF_PREDICT_TRUE GOOGLE_PREDICT_TRUE +#else +#ifdef __GNUC__ +// Provided at least since GCC 3.0. +#define PROTOBUF_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) +#else +#define PROTOBUF_PREDICT_TRUE(x) (x) +#endif +#endif + +#ifdef GOOGLE_PREDICT_FALSE +#define PROTOBUF_PREDICT_FALSE GOOGLE_PREDICT_FALSE +#else +#ifdef __GNUC__ +// Provided at least since GCC 3.0. +#define PROTOBUF_PREDICT_FALSE(x) (__builtin_expect(x, 0)) +#else +#define PROTOBUF_PREDICT_FALSE(x) (x) +#endif +#endif + +#ifdef GOOGLE_PROTOBUF_ATTRIBUTE_RETURNS_NONNULL +#define PROTOBUF_RETURNS_NONNULL GOOGLE_PROTOBUF_ATTRIBUTE_RETURNS_NONNULL +#else +#ifdef __GNUC__ +#define PROTOBUF_RETURNS_NONNULL __attribute__((returns_nonnull)) +#else +#define PROTOBUF_RETURNS_NONNULL +#endif +#endif + +#if defined(__has_cpp_attribute) +#if __has_cpp_attribute(clang::reinitializes) +#define PROTOBUF_ATTRIBUTE_REINITIALIZES [[clang::reinitializes]] +#endif +#endif +#ifndef PROTOBUF_ATTRIBUTE_REINITIALIZES +#define PROTOBUF_ATTRIBUTE_REINITIALIZES +#endif + +#define PROTOBUF_GUARDED_BY(x) +#define PROTOBUF_COLD + +// Copied from ABSL. +#if defined(__clang__) && defined(__has_warning) +#if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#define PROTOBUF_FALLTHROUGH_INTENDED [[clang::fallthrough]] +#endif +#elif defined(__GNUC__) && __GNUC__ >= 7 +#define PROTOBUF_FALLTHROUGH_INTENDED [[gnu::fallthrough]] +#endif + +#ifndef PROTOBUF_FALLTHROUGH_INTENDED +#define PROTOBUF_FALLTHROUGH_INTENDED +#endif + +#if defined(__has_cpp_attribute) +#define HAS_ATTRIBUTE(attr) __has_cpp_attribute(attr) +#else +#define HAS_ATTRIBUTE(attr) 0 +#endif + +#if HAS_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) +#define PROTOBUF_UNUSED __attribute__((__unused__)) +#else +#define PROTOBUF_UNUSED +#endif + +#undef HAS_ATTRIBUTE + +#ifdef _MSC_VER +#define PROTOBUF_LONGLONG(x) x##I64 +#define PROTOBUF_ULONGLONG(x) x##UI64 +#define PROTOBUF_LL_FORMAT "I64" // As in printf("%I64d", ...) +#else +// By long long, we actually mean int64. +#define PROTOBUF_LONGLONG(x) x##LL +#define PROTOBUF_ULONGLONG(x) x##ULL +// Used to format real long long integers. +#define PROTOBUF_LL_FORMAT \ + "ll" // As in "%lld". Note that "q" is poor form also. +#endif + + +// Shared google3/opensource definitions. ////////////////////////////////////// + +#define PROTOBUF_VERSION 3012003 +#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3012000 +#define PROTOBUF_MIN_PROTOC_VERSION 3012000 +#define PROTOBUF_VERSION_SUFFIX "" + +// The minimum library version which works with the current version of the +// headers. +#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3012000 + +#if defined(GOOGLE_PROTOBUF_NO_RTTI) && GOOGLE_PROTOBUF_NO_RTTI +#define PROTOBUF_RTTI 0 +#elif defined(__has_feature) +// https://clang.llvm.org/docs/LanguageExtensions.html#has-feature-and-has-extension +#define PROTOBUF_RTTI __has_feature(cxx_rtti) +#elif !defined(__cxx_rtti) +// https://en.cppreference.com/w/User:D41D8CD98F/feature_testing_macros#C.2B.2B98 +#define PROTOBUF_RTTI 0 +#elif defined(__GNUC__) && !defined(__GXX_RTTI) +#https: // gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html +#define PROTOBUF_RTTI 0 +#else +#define PROTOBUF_RTTI 1 +#endif + +// Returns the offset of the given field within the given aggregate type. +// This is equivalent to the ANSI C offsetof() macro. However, according +// to the C++ standard, offsetof() only works on POD types, and GCC +// enforces this requirement with a warning. In practice, this rule is +// unnecessarily strict; there is probably no compiler or platform on +// which the offsets of the direct fields of a class are non-constant. +// Fields inherited from superclasses *can* have non-constant offsets, +// but that's not what this macro will be used for. +#if defined(__clang__) +// For Clang we use __builtin_offsetof() and suppress the warning, +// to avoid Control Flow Integrity and UBSan vptr sanitizers from +// crashing while trying to validate the invalid reinterpet_casts. +#define PROTOBUF_FIELD_OFFSET(TYPE, FIELD) \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \ + __builtin_offsetof(TYPE, FIELD) \ + _Pragma("clang diagnostic pop") +#elif defined(__GNUC__) && \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) +#define PROTOBUF_FIELD_OFFSET(TYPE, FIELD) __builtin_offsetof(TYPE, FIELD) +#else // defined(__clang__) +// Note that we calculate relative to the pointer value 16 here since if we +// just use zero, GCC complains about dereferencing a NULL pointer. We +// choose 16 rather than some other number just in case the compiler would +// be confused by an unaligned pointer. +#define PROTOBUF_FIELD_OFFSET(TYPE, FIELD) \ + static_cast< ::google::protobuf::uint32>(reinterpret_cast( \ + &reinterpret_cast(16)->FIELD) - \ + reinterpret_cast(16)) +#endif + +#if defined(PROTOBUF_USE_DLLS) + #if defined(_MSC_VER) + #ifdef LIBPROTOBUF_EXPORTS + #define PROTOBUF_EXPORT __declspec(dllexport) + #define PROTOBUF_EXPORT_TEMPLATE_DECLARE + #define PROTOBUF_EXPORT_TEMPLATE_DEFINE __declspec(dllexport) + #else + #define PROTOBUF_EXPORT __declspec(dllimport) + #define PROTOBUF_EXPORT_TEMPLATE_DECLARE + #define PROTOBUF_EXPORT_TEMPLATE_DEFINE __declspec(dllimport) + #endif + #ifdef LIBPROTOC_EXPORTS + #define PROTOC_EXPORT __declspec(dllexport) + #else + #define PROTOC_EXPORT __declspec(dllimport) + #endif + #else // defined(_MSC_VER) + #ifdef LIBPROTOBUF_EXPORTS + #define PROTOBUF_EXPORT __attribute__((visibility("default"))) + #define PROTOBUF_EXPORT_TEMPLATE_DECLARE __attribute__((visibility("default"))) + #define PROTOBUF_EXPORT_TEMPLATE_DEFINE + #else + #define PROTOBUF_EXPORT + #define PROTOBUF_EXPORT_TEMPLATE_DECLARE + #define PROTOBUF_EXPORT_TEMPLATE_DEFINE + #endif + #ifdef LIBPROTOC_EXPORTS + #define PROTOC_EXPORT __attribute__((visibility("default"))) + #else + #define PROTOC_EXPORT + #endif + #endif +#else // defined(PROTOBUF_USE_DLLS) + #define PROTOBUF_EXPORT + #define PROTOC_EXPORT + #define PROTOBUF_EXPORT_TEMPLATE_DECLARE + #define PROTOBUF_EXPORT_TEMPLATE_DEFINE +#endif + +// Windows declares several inconvenient macro names. We #undef them and then +// restore them in port_undef.inc. +#ifdef _MSC_VER +#pragma push_macro("ERROR") +#undef ERROR +#pragma push_macro("GetMessage") +#undef GetMessage +#pragma push_macro("IGNORE") +#undef IGNORE +#pragma push_macro("IN") +#undef IN +#pragma push_macro("OUT") +#undef OUT +#pragma push_macro("OPTIONAL") +#undef OPTIONAL +#pragma push_macro("min") +#undef min +#pragma push_macro("max") +#undef max +#endif // _MSC_VER + +#if defined(__clang__) || defined(__GNUC__) || defined(_MSC_VER) +// Don't let the YES/NO Objective-C Macros interfere with proto identifiers with +// the same name. +#pragma push_macro("YES") +#undef YES +#pragma push_macro("NO") +#undef NO +#endif // defined(__clang__) || defined(__GNUC__) || defined(_MSC_VER) + +#if defined(__clang__) +#pragma clang diagnostic push +// TODO(gerbens) ideally we cleanup the code. But a cursory try shows many +// violations. So let's ignore for now. +#pragma clang diagnostic ignored "-Wshorten-64-to-32" +#elif defined(__GNUC__) +// GCC does not allow disabling diagnostics within an expression: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=60875, so we disable this one +// globally even though it's only used for PROTOBUF_FIELD_OFFSET. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#endif + +// PROTOBUF_ASSUME(pred) tells the compiler that it can assume pred is true. To +// be safe, we also validate the assumption with a GOOGLE_DCHECK in unoptimized +// builds. The macro does not do anything useful if the compiler does not +// support __builtin_assume. +#ifdef __has_builtin +#if __has_builtin(__builtin_assume) +#define PROTOBUF_ASSUME(pred) \ + GOOGLE_DCHECK(pred); \ + __builtin_assume(pred) +#else +#define PROTOBUF_ASSUME(pred) GOOGLE_DCHECK(pred) +#endif +#else +#define PROTOBUF_ASSUME(pred) GOOGLE_DCHECK(pred) +#endif + +// Specify memory alignment for structs, classes, etc. +// Use like: +// class PROTOBUF_ALIGNAS(16) MyClass { ... } +// PROTOBUF_ALIGNAS(16) int array[4]; +// +// In most places you can use the C++11 keyword "alignas", which is preferred. +// +// But compilers have trouble mixing __attribute__((...)) syntax with +// alignas(...) syntax. +// +// Doesn't work in clang or gcc: +// struct alignas(16) __attribute__((packed)) S { char c; }; +// Works in clang but not gcc: +// struct __attribute__((packed)) alignas(16) S2 { char c; }; +// Works in clang and gcc: +// struct alignas(16) S3 { char c; } __attribute__((packed)); +// +// There are also some attributes that must be specified *before* a class +// definition: visibility (used for exporting functions/classes) is one of +// these attributes. This means that it is not possible to use alignas() with a +// class that is marked as exported. +#if defined(_MSC_VER) +#define PROTOBUF_ALIGNAS(byte_alignment) __declspec(align(byte_alignment)) +#elif defined(__GNUC__) +#define PROTOBUF_ALIGNAS(byte_alignment) \ + __attribute__((aligned(byte_alignment))) +#else +#define PROTOBUF_ALIGNAS(byte_alignment) alignas(byte_alignment) +#endif + +#define PROTOBUF_FINAL final diff --git a/third_party/protobuf-lite/google/protobuf/port_undef.inc b/third_party/protobuf-lite/google/protobuf/port_undef.inc new file mode 100644 index 00000000..b35e8568 --- /dev/null +++ b/third_party/protobuf-lite/google/protobuf/port_undef.inc @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// #undefs all macros defined in port_def.inc. See comments in port_def.inc +// for more info. + +#ifndef PROTOBUF_NAMESPACE +#error "port_undef.inc must be included after port_def.inc" +#endif +#undef PROTOBUF_NAMESPACE +#undef PROTOBUF_NAMESPACE_ID +#undef PROTOBUF_ALWAYS_INLINE +#undef PROTOBUF_COLD +#undef PROTOBUF_NOINLINE +#undef PROTOBUF_SECTION_VARIABLE +#undef PROTOBUF_DEPRECATED +#undef PROTOBUF_DEPRECATED_ENUM +#undef PROTOBUF_DEPRECATED_MSG +#undef PROTOBUF_FUNC_ALIGN +#undef PROTOBUF_RETURNS_NONNULL +#undef PROTOBUF_ATTRIBUTE_REINITIALIZES +#undef PROTOBUF_RTTI +#undef PROTOBUF_VERSION +#undef PROTOBUF_VERSION_SUFFIX +#undef PROTOBUF_FIELD_OFFSET +#undef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC +#undef PROTOBUF_MIN_PROTOC_VERSION +#undef PROTOBUF_PREDICT_TRUE +#undef PROTOBUF_PREDICT_FALSE +#undef PROTOBUF_LONGLONG +#undef PROTOBUF_ULONGLONG +#undef PROTOBUF_LL_FORMAT +#undef PROTOBUF_GUARDED_BY +#undef PROTOBUF_FALLTHROUGH_INTENDED +#undef PROTOBUF_EXPORT +#undef PROTOC_EXPORT +#undef PROTOBUF_MUST_USE_RESULT +#undef PROTOBUF_NAMESPACE_OPEN +#undef PROTOBUF_NAMESPACE_CLOSE +#undef PROTOBUF_UNUSED +#undef PROTOBUF_ASSUME +#undef PROTOBUF_EXPORT_TEMPLATE_DECLARE +#undef PROTOBUF_EXPORT_TEMPLATE_DEFINE +#undef PROTOBUF_ALIGNAS +#undef PROTOBUF_FINAL + +// Restore macro that may have been #undef'd in port_def.inc. +#ifdef _MSC_VER +#pragma pop_macro("ERROR") +#pragma pop_macro("GetMessage") +#pragma pop_macro("IGNORE") +#pragma pop_macro("IN") +#pragma pop_macro("OUT") +#pragma pop_macro("OPTIONAL") +#pragma pop_macro("min") +#pragma pop_macro("max") +#endif + +#if defined(__clang__) || defined(__GNUC__) || defined(_MSC_VER) +#pragma pop_macro("YES") +#pragma pop_macro("NO") +#endif // defined(__clang__) || defined(__GNUC__) || defined(_MSC_VER) + +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif diff --git a/third_party/protobuf-lite/google/protobuf/repeated_field.h b/third_party/protobuf-lite/google/protobuf/repeated_field.h index b47ea994..fd01a667 100644 --- a/third_party/protobuf-lite/google/protobuf/repeated_field.h +++ b/third_party/protobuf-lite/google/protobuf/repeated_field.h @@ -46,6 +46,7 @@ #ifndef GOOGLE_PROTOBUF_REPEATED_FIELD_H__ #define GOOGLE_PROTOBUF_REPEATED_FIELD_H__ +#include #ifdef _MSC_VER // This is required for min/max on VS2013 only. #include @@ -54,33 +55,47 @@ #include #include #include -#include +#include + #include #include #include -#include #include -#include +#include +#include #include -// Forward-declare these so that we can make them friends. -namespace google { -namespace upb { -namespace google_opensource { -class GMR_Handlers; -} // namespace google_opensource -} // namespace upb +// Must be included last. +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif +namespace google { namespace protobuf { class Message; +class Reflection; + +template +struct WeakRepeatedPtrField; namespace internal { class MergePartialFromCodedStreamHelper; -static const int kMinRepeatedFieldAllocationSize = 4; +// kRepeatedFieldLowerClampLimit is the smallest size that will be allocated +// when growing a repeated field. +constexpr int kRepeatedFieldLowerClampLimit = 4; + +// kRepeatedFieldUpperClampLimit is the lowest signed integer value that +// overflows when multiplied by 2 (which is undefined behavior). Sizes above +// this will clamp to the maximum int value instead of following exponential +// growth when growing a repeated field. +constexpr int kRepeatedFieldUpperClampLimit = + (std::numeric_limits::max() / 2) + 1; // A utility function for logging that doesn't need any template types. void LogIndexOutOfBounds(int index, int size); @@ -101,8 +116,47 @@ inline int CalculateReserve(Iter begin, Iter end) { typedef typename std::iterator_traits::iterator_category Category; return CalculateReserve(begin, end, Category()); } -} // namespace internal +// Swaps two blocks of memory of size sizeof(T). +template +inline void SwapBlock(char* p, char* q) { + T tmp; + memcpy(&tmp, p, sizeof(T)); + memcpy(p, q, sizeof(T)); + memcpy(q, &tmp, sizeof(T)); +} + +// Swaps two blocks of memory of size kSize: +// template void memswap(char* p, char* q); + +template +inline typename std::enable_if<(kSize == 0), void>::type memswap(char*, char*) { +} + +#define PROTO_MEMSWAP_DEF_SIZE(reg_type, max_size) \ + template \ + typename std::enable_if<(kSize >= sizeof(reg_type) && kSize < (max_size)), \ + void>::type \ + memswap(char* p, char* q) { \ + SwapBlock(p, q); \ + memswap(p + sizeof(reg_type), \ + q + sizeof(reg_type)); \ + } + +PROTO_MEMSWAP_DEF_SIZE(uint8, 2) +PROTO_MEMSWAP_DEF_SIZE(uint16, 4) +PROTO_MEMSWAP_DEF_SIZE(uint32, 8) + +#ifdef __SIZEOF_INT128__ +PROTO_MEMSWAP_DEF_SIZE(uint64, 16) +PROTO_MEMSWAP_DEF_SIZE(__uint128_t, (1u << 31)) +#else +PROTO_MEMSWAP_DEF_SIZE(uint64, (1u << 31)) +#endif + +#undef PROTO_MEMSWAP_DEF_SIZE + +} // namespace internal // RepeatedField is used to represent repeated fields of a primitive type (in // other words, everything except strings and nested Messages). Most users will @@ -110,6 +164,10 @@ inline int CalculateReserve(Iter begin, Iter end) { // set-by-index, and add accessors that are generated for all repeated fields. template class RepeatedField final { + static_assert( + alignof(Arena) >= alignof(Element), + "We only support types that have an alignment smaller than Arena"); + public: RepeatedField(); explicit RepeatedField(Arena* arena); @@ -132,11 +190,19 @@ class RepeatedField final { const Element& operator[](int index) const { return Get(index); } Element& operator[](int index) { return *Mutable(index); } + const Element& at(int index) const; + Element& at(int index); + void Set(int index, const Element& value); void Add(const Element& value); // Appends a new element and return a pointer to it. // The new element is uninitialized if |Element| is a POD type. Element* Add(); + // Append elements in the range [begin, end) after reserving + // the appropriate number of elements. + template + void Add(Iter begin, Iter end); + // Remove the last element in the array. void RemoveLast(); @@ -209,15 +275,11 @@ class RepeatedField final { // Reverse iterator support typedef std::reverse_iterator const_reverse_iterator; typedef std::reverse_iterator reverse_iterator; - reverse_iterator rbegin() { - return reverse_iterator(end()); - } + reverse_iterator rbegin() { return reverse_iterator(end()); } const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } - reverse_iterator rend() { - return reverse_iterator(begin()); - } + reverse_iterator rend() { return reverse_iterator(begin()); } const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } @@ -246,8 +308,9 @@ class RepeatedField final { iterator erase(const_iterator first, const_iterator last); // Get the Arena on which this RepeatedField stores its elements. - ::google::protobuf::Arena* GetArena() const { - return GetArenaNoVirtual(); + inline Arena* GetArena() const { + return (total_size_ == 0) ? static_cast(arena_or_elements_) + : rep()->arena; } // For internal use only. @@ -256,7 +319,7 @@ class RepeatedField final { inline void InternalSwap(RepeatedField* other); private: - static const int kInitialSize = 0; + static constexpr int kInitialSize = 0; // A note on the representation here (see also comment below for // RepeatedPtrFieldBase's struct Rep): // @@ -279,23 +342,36 @@ class RepeatedField final { // Element is double and pointer is 32bit). static const size_t kRepHeaderSize; - // We reuse the Rep* for an Arena* when total_size == 0, to avoid having to do - // an allocation in the constructor when we have an Arena. - union Pointer { - Pointer(Arena* a) : arena(a) {} - Arena* arena; // When total_size_ == 0. - Rep* rep; // When total_size_ != 0. - } ptr_; + // If total_size_ == 0 this points to an Arena otherwise it points to the + // elements member of a Rep struct. Using this invariant allows the storage of + // the arena pointer without an extra allocation in the constructor. + void* arena_or_elements_; - Rep* rep() const { + // Return pointer to elements array. + // pre-condition: the array must have been allocated. + Element* elements() const { GOOGLE_DCHECK_GT(total_size_, 0); - return ptr_.rep; + // Because of above pre-condition this cast is safe. + return unsafe_elements(); + } + + // Return pointer to elements array if it exists otherwise either null or + // a invalid pointer is returned. This only happens for empty repeated fields, + // where you can't dereference this pointer anyway (it's empty). + Element* unsafe_elements() const { + return static_cast(arena_or_elements_); + } + + // Return pointer to the Rep struct. + // pre-condition: the Rep must have been allocated, ie elements() is safe. + Rep* rep() const { + char* addr = reinterpret_cast(elements()) - offsetof(Rep, elements); + return reinterpret_cast(addr); } friend class Arena; typedef void InternalArenaConstructable_; - // Move the contents of |from| into |to|, possibly clobbering |from| in the // process. For primitive types this is just a memcpy(), but it could be // specialized for non-primitive types to, say, swap each element instead. @@ -304,11 +380,6 @@ class RepeatedField final { // Copy the elements of |from| into |to|. void CopyArray(Element* to, const Element* from, int size); - // Internal helper expected by Arena methods. - inline Arena* GetArenaNoVirtual() const { - return (total_size_ == 0) ? ptr_.arena : ptr_.rep->arena; - } - // Internal helper to delete all elements and deallocate the storage. // If Element has a trivial destructor (for example, if it's a fundamental // type, like int32), the loop will be removed by the optimizer. @@ -330,17 +401,94 @@ class RepeatedField final { } } - friend class internal::WireFormatLite; - const Element* unsafe_data() const; + // This class is a performance wrapper around RepeatedField::Add(const T&) + // function. In general unless a RepeatedField is a local stack variable LLVM + // has a hard time optimizing Add. The machine code tends to be + // loop: + // mov %size, dword ptr [%repeated_field] // load + // cmp %size, dword ptr [%repeated_field + 4] + // jae fallback + // mov %buffer, qword ptr [%repeated_field + 8] + // mov dword [%buffer + %size * 4], %value + // inc %size // increment + // mov dword ptr [%repeated_field], %size // store + // jmp loop + // + // This puts a load/store in each iteration of the important loop variable + // size. It's a pretty bad compile that happens even in simple cases, but + // largely the presence of the fallback path disturbs the compilers mem-to-reg + // analysis. + // + // This class takes ownership of a repeated field for the duration of it's + // lifetime. The repeated field should not be accessed during this time, ie. + // only access through this class is allowed. This class should always be a + // function local stack variable. Intended use + // + // void AddSequence(const int* begin, const int* end, RepeatedField* out) + // { + // RepeatedFieldAdder adder(out); // Take ownership of out + // for (auto it = begin; it != end; ++it) { + // adder.Add(*it); + // } + // } + // + // Typically due to the fact adder is a local stack variable. The compiler + // will be successful in mem-to-reg transformation and the machine code will + // be loop: cmp %size, %capacity jae fallback mov dword ptr [%buffer + %size * + // 4], %val inc %size jmp loop + // + // The first version executes at 7 cycles per iteration while the second + // version near 1 or 2 cycles. + class FastAdder { + public: + explicit FastAdder(RepeatedField* rf) : repeated_field_(rf) { + if (kIsPod) { + index_ = repeated_field_->current_size_; + capacity_ = repeated_field_->total_size_; + buffer_ = repeated_field_->unsafe_elements(); + } + } + ~FastAdder() { + if (kIsPod) repeated_field_->current_size_ = index_; + } + + void Add(const Element& val) { + if (kIsPod) { + if (index_ == capacity_) { + repeated_field_->current_size_ = index_; + repeated_field_->Reserve(index_ + 1); + capacity_ = repeated_field_->total_size_; + buffer_ = repeated_field_->unsafe_elements(); + } + buffer_[index_++] = val; + } else { + repeated_field_->Add(val); + } + } + + private: + constexpr static bool kIsPod = std::is_pod::value; + RepeatedField* repeated_field_; + int index_; + int capacity_; + Element* buffer_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FastAdder); + }; + + friend class TestRepeatedFieldHelper; + friend class ::google::protobuf::internal::ParseContext; }; -template +template const size_t RepeatedField::kRepHeaderSize = reinterpret_cast(&reinterpret_cast(16)->elements[0]) - 16; namespace internal { -template class RepeatedPtrIterator; -template class RepeatedPtrOverPtrsIterator; +template +class RepeatedPtrIterator; +template +class RepeatedPtrOverPtrsIterator; } // namespace internal namespace internal { @@ -364,7 +512,7 @@ namespace internal { // arena-related "copy if on different arena" behavior if the necessary methods // exist on the contained type. In particular, we rely on MergeFrom() existing // as a general proxy for the fact that a copy will work, and we also provide a -// specific override for string*. +// specific override for std::string*. template struct TypeImplementsMergeBehaviorProbeForMergeFrom { typedef char HasMerge; @@ -377,29 +525,36 @@ struct TypeImplementsMergeBehaviorProbeForMergeFrom { // We mangle these names a bit to avoid compatibility issues in 'unclean' // include environments that may have, e.g., "#define test ..." (yes, this // exists). - template - struct CheckType; - template static HasMerge Check( - CheckType*); - template static HasMerge Check( - CheckType*); - template static HasNoMerge Check(...); + template + struct CheckType; + template + static HasMerge Check(CheckType*); + template + static HasMerge Check(CheckType*); + template + static HasNoMerge Check(...); // Resolves to either std::true_type or std::false_type. typedef std::integral_constant(0)) == sizeof(HasMerge))> type; + (sizeof(Check(0)) == sizeof(HasMerge))> + type; }; template -struct TypeImplementsMergeBehavior : - TypeImplementsMergeBehaviorProbeForMergeFrom {}; +struct TypeImplementsMergeBehavior + : TypeImplementsMergeBehaviorProbeForMergeFrom {}; template <> -struct TypeImplementsMergeBehavior< ::std::string> { +struct TypeImplementsMergeBehavior { typedef std::true_type type; }; +template +struct IsMovable + : std::integral_constant::value && + std::is_move_assignable::value> {}; + // This is the common base class for RepeatedPtrFields. It deals only in void* // pointers. Users should not use this interface directly. // @@ -408,12 +563,9 @@ struct TypeImplementsMergeBehavior< ::std::string> { // class TypeHandler { // public: // typedef MyType Type; -// // WeakType is almost always the same as MyType, but we use it in -// // ImplicitWeakTypeHandler. -// typedef MyType WeakType; // static Type* New(); -// static WeakType* NewFromPrototype(const WeakType* prototype, -// ::google::protobuf::Arena* arena); +// static Type* NewFromPrototype(const Type* prototype, +// Arena* arena); // static void Delete(Type*); // static void Clear(Type*); // static void Merge(const Type& from, Type* to); @@ -421,19 +573,32 @@ struct TypeImplementsMergeBehavior< ::std::string> { // // Only needs to be implemented if SpaceUsedExcludingSelf() is called. // static int SpaceUsedLong(const Type&); // }; -class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { +class PROTOBUF_EXPORT RepeatedPtrFieldBase { protected: RepeatedPtrFieldBase(); - explicit RepeatedPtrFieldBase(::google::protobuf::Arena* arena); - ~RepeatedPtrFieldBase() {} + explicit RepeatedPtrFieldBase(Arena* arena); + ~RepeatedPtrFieldBase() { +#ifndef NDEBUG + // Try to trigger segfault / asan failure in non-opt builds. If arena_ + // lifetime has ended before the destructor. + if (arena_) (void)arena_->SpaceAllocated(); +#endif + } + public: // Must be called from destructor. template void Destroy(); + protected: bool empty() const; int size() const; + template + const typename TypeHandler::Type& at(int index) const; + template + typename TypeHandler::Type& at(int index); + template typename TypeHandler::Type* Mutable(int index); template @@ -447,13 +612,13 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { // application code. template - const typename TypeHandler::WeakType& Get(int index) const; + const typename TypeHandler::Type& Get(int index) const; // Creates and adds an element using the given prototype, without introducing // a link-time dependency on the concrete message type. This method is used to // implement implicit weak fields. The prototype may be NULL, in which case an // ImplicitWeakMessage will be used as a placeholder. - google::protobuf::MessageLite* AddWeak(const google::protobuf::MessageLite* prototype); + MessageLite* AddWeak(const MessageLite* prototype); template void Clear(); @@ -464,9 +629,10 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { inline void InternalSwap(RepeatedPtrFieldBase* other); protected: - template - void Add(typename TypeHandler::Type&& value, - std::enable_if* dummy = NULL); + template < + typename TypeHandler, + typename std::enable_if::type* = nullptr> + void Add(typename TypeHandler::Type&& value); template void RemoveLast(); @@ -488,8 +654,8 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { template const typename TypeHandler::Type* const* data() const; - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - void Swap(RepeatedPtrFieldBase* other); + template + PROTOBUF_ALWAYS_INLINE void Swap(RepeatedPtrFieldBase* other); void SwapElements(int index1, int index2); @@ -502,7 +668,7 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { template typename TypeHandler::Type* AddFromCleared(); - template + template void AddAllocated(typename TypeHandler::Type* value) { typename TypeImplementsMergeBehavior::type t; AddAllocatedInternal(value, t); @@ -533,27 +699,25 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { template void AddAllocatedInternal(typename TypeHandler::Type* value, std::false_type); - template GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void AddAllocatedSlowWithCopy(typename TypeHandler::Type* value, - Arena* value_arena, - Arena* my_arena); - template GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void AddAllocatedSlowWithoutCopy(typename TypeHandler::Type* value); + template + PROTOBUF_NOINLINE void AddAllocatedSlowWithCopy( + typename TypeHandler::Type* value, Arena* value_arena, Arena* my_arena); + template + PROTOBUF_NOINLINE void AddAllocatedSlowWithoutCopy( + typename TypeHandler::Type* value); template typename TypeHandler::Type* ReleaseLastInternal(std::true_type); template typename TypeHandler::Type* ReleaseLastInternal(std::false_type); - template GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void SwapFallback(RepeatedPtrFieldBase* other); + template + PROTOBUF_NOINLINE void SwapFallback(RepeatedPtrFieldBase* other); - inline Arena* GetArenaNoVirtual() const { - return arena_; - } + inline Arena* GetArena() const { return arena_; } private: - static const int kInitialSize = 0; + static constexpr int kInitialSize = 0; // A few notes on internal representation: // // We use an indirected approach, with struct Rep, to keep @@ -566,15 +730,13 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { // Placing all fields directly in the RepeatedPtrFieldBase instance costs // significant performance for memory-sensitive workloads. Arena* arena_; - int current_size_; - int total_size_; + int current_size_; + int total_size_; struct Rep { - int allocated_size; - void* elements[1]; + int allocated_size; + void* elements[1]; }; - static const size_t kRepHeaderSize = sizeof(Rep) - sizeof(void*); - // Contains arena ptr and the elements array. We also keep the invariant that - // if rep_ is NULL, then arena is NULL. + static constexpr size_t kRepHeaderSize = sizeof(Rep) - sizeof(void*); Rep* rep_; template @@ -588,13 +750,14 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { // Non-templated inner function to avoid code duplication. Takes a function // pointer to the type-specific (templated) inner allocate/merge loop. - void MergeFromInternal( - const RepeatedPtrFieldBase& other, - void (RepeatedPtrFieldBase::*inner_loop)(void**, void**, int, int)); + void MergeFromInternal(const RepeatedPtrFieldBase& other, + void (RepeatedPtrFieldBase::*inner_loop)(void**, + void**, int, + int)); - template - void MergeFromInnerLoop( - void** our_elems, void** other_elems, int length, int already_allocated); + template + void MergeFromInnerLoop(void** our_elems, void** other_elems, int length, + int already_allocated); // Internal helper: extend array space if necessary to contain |extend_amount| // more elements, and return a pointer to the element immediately following @@ -605,7 +768,7 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { // The reflection implementation needs to call protected methods directly, // reinterpreting pointers as being to Message instead of a specific Message // subclass. - friend class GeneratedMessageReflection; + friend class ::PROTOBUF_NAMESPACE_ID::Reflection; // ExtensionSet stores repeated message extensions as // RepeatedPtrField, but non-lite ExtensionSets need to implement @@ -622,12 +785,9 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { // The table-driven MergePartialFromCodedStream implementation needs to // operate on RepeatedPtrField. friend class MergePartialFromCodedStreamHelper; - - // To parse directly into a proto2 generated class, the upb class GMR_Handlers - // needs to be able to modify a RepeatedPtrFieldBase directly. - friend class upb::google_opensource::GMR_Handlers; - friend class AccessorHelper; + template + friend struct google::protobuf::WeakRepeatedPtrField; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(RepeatedPtrFieldBase); }; @@ -636,28 +796,30 @@ template class GenericTypeHandler { public: typedef GenericType Type; - typedef GenericType WeakType; - static const bool Moveable = false; + using Movable = IsMovable; static inline GenericType* New(Arena* arena) { - return ::google::protobuf::Arena::CreateMaybeMessage(arena); + return Arena::CreateMaybeMessage(arena); + } + static inline GenericType* New(Arena* arena, GenericType&& value) { + return Arena::Create(arena, std::move(value)); } - static inline GenericType* NewFromPrototype( - const GenericType* prototype, ::google::protobuf::Arena* arena = NULL); + static inline GenericType* NewFromPrototype(const GenericType* prototype, + Arena* arena = NULL); static inline void Delete(GenericType* value, Arena* arena) { if (arena == NULL) { delete value; } } - static inline ::google::protobuf::Arena* GetArena(GenericType* value) { - return ::google::protobuf::Arena::GetArena(value); + static inline Arena* GetArena(GenericType* value) { + return Arena::GetArena(value); } static inline void* GetMaybeArenaPointer(GenericType* value) { - return ::google::protobuf::Arena::GetArena(value); + return Arena::GetArena(value); } static inline void Clear(GenericType* value) { value->Clear(); } - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE + PROTOBUF_NOINLINE static void Merge(const GenericType& from, GenericType* to); static inline size_t SpaceUsedLong(const GenericType& value) { return value.SpaceUsedLong(); @@ -666,7 +828,7 @@ class GenericTypeHandler { template GenericType* GenericTypeHandler::NewFromPrototype( - const GenericType* /* prototype */, ::google::protobuf::Arena* arena) { + const GenericType* /* prototype */, Arena* arena) { return New(arena); } template @@ -678,15 +840,14 @@ void GenericTypeHandler::Merge(const GenericType& from, // NewFromPrototype() and Merge() are not defined inline here, as we will need // to do a virtual function dispatch anyways to go from Message* to call // New/Merge. -template<> +template <> MessageLite* GenericTypeHandler::NewFromPrototype( - const MessageLite* prototype, google::protobuf::Arena* arena); -template<> -inline google::protobuf::Arena* GenericTypeHandler::GetArena( - MessageLite* value) { + const MessageLite* prototype, Arena* arena); +template <> +inline Arena* GenericTypeHandler::GetArena(MessageLite* value) { return value->GetArena(); } -template<> +template <> inline void* GenericTypeHandler::GetMaybeArenaPointer( MessageLite* value) { return value->GetMaybeArenaPointer(); @@ -694,66 +855,55 @@ inline void* GenericTypeHandler::GetMaybeArenaPointer( template <> void GenericTypeHandler::Merge(const MessageLite& from, MessageLite* to); -template<> -inline void GenericTypeHandler::Clear(string* value) { +template <> +inline void GenericTypeHandler::Clear(std::string* value) { value->clear(); } -template<> -void GenericTypeHandler::Merge(const string& from, - string* to); - -// Declarations of the specialization as we cannot define them here, as the -// header that defines ProtocolMessage depends on types defined in this header. -#define DECLARE_SPECIALIZATIONS_FOR_BASE_PROTO_TYPES(TypeName) \ - template<> \ - TypeName* GenericTypeHandler::NewFromPrototype( \ - const TypeName* prototype, google::protobuf::Arena* arena); \ - template<> \ - google::protobuf::Arena* GenericTypeHandler::GetArena( \ - TypeName* value); \ - template<> \ - void* GenericTypeHandler::GetMaybeArenaPointer( \ - TypeName* value); +template <> +void GenericTypeHandler::Merge(const std::string& from, + std::string* to); // Message specialization bodies defined in message.cc. This split is necessary // to allow proto2-lite (which includes this header) to be independent of // Message. -DECLARE_SPECIALIZATIONS_FOR_BASE_PROTO_TYPES(Message) - - -#undef DECLARE_SPECIALIZATIONS_FOR_BASE_PROTO_TYPES +template <> +PROTOBUF_EXPORT Message* GenericTypeHandler::NewFromPrototype( + const Message* prototype, Arena* arena); +template <> +PROTOBUF_EXPORT Arena* GenericTypeHandler::GetArena(Message* value); +template <> +PROTOBUF_EXPORT void* GenericTypeHandler::GetMaybeArenaPointer( + Message* value); class StringTypeHandler { public: - typedef string Type; - typedef string WeakType; - static const bool Moveable = std::is_move_constructible::value && - std::is_move_assignable::value; + typedef std::string Type; + using Movable = IsMovable; - static inline string* New(Arena* arena) { - return Arena::Create(arena); + static inline std::string* New(Arena* arena) { + return Arena::Create(arena); } - static inline string* New(Arena* arena, string&& value) { - return Arena::Create(arena, std::move(value)); + static inline std::string* New(Arena* arena, std::string&& value) { + return Arena::Create(arena, std::move(value)); } - static inline string* NewFromPrototype(const string*, - ::google::protobuf::Arena* arena) { + static inline std::string* NewFromPrototype(const std::string*, + Arena* arena) { return New(arena); } - static inline ::google::protobuf::Arena* GetArena(string*) { - return NULL; - } - static inline void* GetMaybeArenaPointer(string* /* value */) { + static inline Arena* GetArena(std::string*) { return NULL; } + static inline void* GetMaybeArenaPointer(std::string* /* value */) { return NULL; } - static inline void Delete(string* value, Arena* arena) { + static inline void Delete(std::string* value, Arena* arena) { if (arena == NULL) { delete value; } } - static inline void Clear(string* value) { value->clear(); } - static inline void Merge(const string& from, string* to) { *to = from; } - static size_t SpaceUsedLong(const string& value) { + static inline void Clear(std::string* value) { value->clear(); } + static inline void Merge(const std::string& from, std::string* to) { + *to = from; + } + static size_t SpaceUsedLong(const std::string& value) { return sizeof(value) + StringSpaceUsedExcludingSelfLong(value); } }; @@ -766,7 +916,7 @@ template class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { public: RepeatedPtrField(); - explicit RepeatedPtrField(::google::protobuf::Arena* arena); + explicit RepeatedPtrField(Arena* arena); RepeatedPtrField(const RepeatedPtrField& other); template @@ -789,6 +939,9 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { const Element& operator[](int index) const { return Get(index); } Element& operator[](int index) { return *Mutable(index); } + const Element& at(int index) const; + Element& at(int index); + // Remove the last element in the array. // Ownership of the element is retained by the array. void RemoveLast(); @@ -848,15 +1001,11 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { // Reverse iterator support typedef std::reverse_iterator const_reverse_iterator; typedef std::reverse_iterator reverse_iterator; - reverse_iterator rbegin() { - return reverse_iterator(end()); - } + reverse_iterator rbegin() { return reverse_iterator(end()); } const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } - reverse_iterator rend() { - return reverse_iterator(begin()); - } + reverse_iterator rend() { return reverse_iterator(begin()); } const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } @@ -992,22 +1141,19 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { iterator erase(const_iterator first, const_iterator last); // Gets the arena on which this RepeatedPtrField stores its elements. - ::google::protobuf::Arena* GetArena() const { - return GetArenaNoVirtual(); - } + inline Arena* GetArena() const; // For internal use only. // // This is public due to it being called by generated code. - using RepeatedPtrFieldBase::InternalSwap; + void InternalSwap(RepeatedPtrField* other) { + internal::RepeatedPtrFieldBase::InternalSwap(other); + } private: // Note: RepeatedPtrField SHOULD NOT be subclassed by users. class TypeHandler; - // Internal arena accessor expected by helpers in Arena. - inline Arena* GetArenaNoVirtual() const; - // Implementations for ExtractSubrange(). The copying behavior must be // included only if the type supports the necessary operations (e.g., // MergeFrom()), so we must resolve this at compile time. ExtractSubrange() @@ -1018,7 +1164,9 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { std::false_type); friend class Arena; - friend class MessageLite; + + template + friend struct WeakRepeatedPtrField; typedef void InternalArenaConstructable_; @@ -1028,23 +1176,15 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { template inline RepeatedField::RepeatedField() - : current_size_(0), - total_size_(0), - ptr_(NULL) { -} + : current_size_(0), total_size_(0), arena_or_elements_(nullptr) {} template inline RepeatedField::RepeatedField(Arena* arena) - : current_size_(0), - total_size_(0), - ptr_(arena) { -} + : current_size_(0), total_size_(0), arena_or_elements_(arena) {} template inline RepeatedField::RepeatedField(const RepeatedField& other) - : current_size_(0), - total_size_(0), - ptr_(NULL) { + : current_size_(0), total_size_(0), arena_or_elements_(nullptr) { if (other.current_size_ != 0) { Reserve(other.size()); AddNAlreadyReserved(other.size()); @@ -1055,20 +1195,8 @@ inline RepeatedField::RepeatedField(const RepeatedField& other) template template RepeatedField::RepeatedField(Iter begin, const Iter& end) - : current_size_(0), - total_size_(0), - ptr_(NULL) { - int reserve = internal::CalculateReserve(begin, end); - if (reserve != -1) { - Reserve(reserve); - for (; begin != end; ++begin) { - AddAlreadyReserved(*begin); - } - } else { - for (; begin != end; ++begin) { - Add(*begin); - } - } + : current_size_(0), total_size_(0), arena_or_elements_(nullptr) { + Add(begin, end); } template @@ -1079,10 +1207,9 @@ RepeatedField::~RepeatedField() { } template -inline RepeatedField& -RepeatedField::operator=(const RepeatedField& other) { - if (this != &other) - CopyFrom(other); +inline RepeatedField& RepeatedField::operator=( + const RepeatedField& other) { + if (this != &other) CopyFrom(other); return *this; } @@ -1090,8 +1217,9 @@ template inline RepeatedField::RepeatedField(RepeatedField&& other) noexcept : RepeatedField() { // We don't just call Swap(&other) here because it would perform 3 copies if - // the two fields are on different arenas. - if (other.GetArenaNoVirtual()) { + // other is on an arena. This field can't be on an arena because arena + // construction always uses the Arena* accepting constructor. + if (other.GetArena()) { CopyFrom(other); } else { InternalSwap(&other); @@ -1104,7 +1232,7 @@ inline RepeatedField& RepeatedField::operator=( // We don't just call Swap(&other) here because it would perform 3 copies if // the two fields are on different arenas. if (this != &other) { - if (this->GetArenaNoVirtual() != other.GetArenaNoVirtual()) { + if (this->GetArena() != other.GetArena()) { CopyFrom(other); } else { InternalSwap(&other); @@ -1128,35 +1256,37 @@ inline int RepeatedField::Capacity() const { return total_size_; } -template +template inline void RepeatedField::AddAlreadyReserved(const Element& value) { GOOGLE_DCHECK_LT(current_size_, total_size_); - rep()->elements[current_size_++] = value; + elements()[current_size_++] = value; } -template +template inline Element* RepeatedField::AddAlreadyReserved() { GOOGLE_DCHECK_LT(current_size_, total_size_); - return &rep()->elements[current_size_++]; + return &elements()[current_size_++]; } -template -inline Element* RepeatedField::AddNAlreadyReserved(int elements) { - GOOGLE_DCHECK_LE(current_size_ + elements, total_size_); - // Warning: total_size_ can be NULL if elements == 0 && current_size_ == 0. - // Existing callers depend on this behavior. :( - Element* ret = &ptr_.rep->elements[current_size_]; - current_size_ += elements; +template +inline Element* RepeatedField::AddNAlreadyReserved(int n) { + GOOGLE_DCHECK_GE(total_size_ - current_size_, n) + << total_size_ << ", " << current_size_; + // Warning: sometimes people call this when n == 0 and total_size_ == 0. In + // this case the return pointer points to a zero size array (n == 0). Hence + // we can just use unsafe_elements(), because the user cannot dereference the + // pointer anyway. + Element* ret = unsafe_elements() + current_size_; + current_size_ += n; return ret; } -template +template inline void RepeatedField::Resize(int new_size, const Element& value) { GOOGLE_DCHECK_GE(new_size, 0); if (new_size > current_size_) { Reserve(new_size); - std::fill(&rep()->elements[current_size_], - &rep()->elements[new_size], value); + std::fill(&elements()[current_size_], &elements()[new_size], value); } current_size_ = new_size; } @@ -1165,33 +1295,76 @@ template inline const Element& RepeatedField::Get(int index) const { GOOGLE_DCHECK_GE(index, 0); GOOGLE_DCHECK_LT(index, current_size_); - return rep()->elements[index]; + return elements()[index]; +} + +template +inline const Element& RepeatedField::at(int index) const { + GOOGLE_CHECK_GE(index, 0); + GOOGLE_CHECK_LT(index, current_size_); + return elements()[index]; +} + +template +inline Element& RepeatedField::at(int index) { + GOOGLE_CHECK_GE(index, 0); + GOOGLE_CHECK_LT(index, current_size_); + return elements()[index]; } template inline Element* RepeatedField::Mutable(int index) { GOOGLE_DCHECK_GE(index, 0); GOOGLE_DCHECK_LT(index, current_size_); - return &rep()->elements[index]; + return &elements()[index]; } template inline void RepeatedField::Set(int index, const Element& value) { GOOGLE_DCHECK_GE(index, 0); GOOGLE_DCHECK_LT(index, current_size_); - rep()->elements[index] = value; + elements()[index] = value; } template inline void RepeatedField::Add(const Element& value) { - if (current_size_ == total_size_) Reserve(total_size_ + 1); - rep()->elements[current_size_++] = value; + uint32 size = current_size_; + if (static_cast(size) == total_size_) Reserve(total_size_ + 1); + elements()[size] = value; + current_size_ = size + 1; } template inline Element* RepeatedField::Add() { - if (current_size_ == total_size_) Reserve(total_size_ + 1); - return &rep()->elements[current_size_++]; + uint32 size = current_size_; + if (static_cast(size) == total_size_) Reserve(total_size_ + 1); + auto ptr = &elements()[size]; + current_size_ = size + 1; + return ptr; +} + +template +template +inline void RepeatedField::Add(Iter begin, Iter end) { + int reserve = internal::CalculateReserve(begin, end); + if (reserve != -1) { + if (reserve == 0) { + return; + } + + Reserve(reserve + size()); + // TODO(ckennelly): The compiler loses track of the buffer freshly + // allocated by Reserve() by the time we call elements, so it cannot + // guarantee that elements does not alias [begin(), end()). + // + // If restrict is available, annotating the pointer obtained from elements() + // causes this to lower to memcpy instead of memmove. + std::copy(begin, end, elements() + size()); + current_size_ = reserve + size(); + } else { + FastAdder fast_adder(this); + for (; begin != end; ++begin) fast_adder.Add(*begin); + } } template @@ -1201,16 +1374,15 @@ inline void RepeatedField::RemoveLast() { } template -void RepeatedField::ExtractSubrange( - int start, int num, Element* elements) { +void RepeatedField::ExtractSubrange(int start, int num, + Element* elements) { GOOGLE_DCHECK_GE(start, 0); GOOGLE_DCHECK_GE(num, 0); GOOGLE_DCHECK_LE(start + num, this->current_size_); // Save the values of the removed elements if requested. if (elements != NULL) { - for (int i = 0; i < num; ++i) - elements[i] = this->Get(i + start); + for (int i = 0; i < num; ++i) elements[i] = this->Get(i + start); } // Slide remaining elements down to fill the gap. @@ -1262,36 +1434,36 @@ inline typename RepeatedField::iterator RepeatedField::erase( template inline Element* RepeatedField::mutable_data() { - return total_size_ > 0 ? rep()->elements : NULL; + return unsafe_elements(); } template inline const Element* RepeatedField::data() const { - return total_size_ > 0 ? rep()->elements : NULL; -} - -template -inline const Element* RepeatedField::unsafe_data() const { - return rep()->elements; + return unsafe_elements(); } template inline void RepeatedField::InternalSwap(RepeatedField* other) { GOOGLE_DCHECK(this != other); - GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); + GOOGLE_DCHECK(GetArena() == other->GetArena()); - std::swap(ptr_, other->ptr_); - std::swap(current_size_, other->current_size_); - std::swap(total_size_, other->total_size_); + // Swap all fields at once. + static_assert(std::is_standard_layout>::value, + "offsetof() requires standard layout before c++17"); + internal::memswaparena_or_elements_) - + offsetof(RepeatedField, current_size_)>( + reinterpret_cast(this) + offsetof(RepeatedField, current_size_), + reinterpret_cast(other) + offsetof(RepeatedField, current_size_)); } template void RepeatedField::Swap(RepeatedField* other) { if (this == other) return; - if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + if (GetArena() == other->GetArena()) { InternalSwap(other); } else { - RepeatedField temp(other->GetArenaNoVirtual()); + RepeatedField temp(other->GetArena()); temp.MergeFrom(*this); CopyFrom(*other); other->UnsafeArenaSwap(&temp); @@ -1307,38 +1479,37 @@ void RepeatedField::UnsafeArenaSwap(RepeatedField* other) { template void RepeatedField::SwapElements(int index1, int index2) { using std::swap; // enable ADL with fallback - swap(rep()->elements[index1], rep()->elements[index2]); + swap(elements()[index1], elements()[index2]); } template inline typename RepeatedField::iterator RepeatedField::begin() { - return total_size_ > 0 ? rep()->elements : NULL; + return unsafe_elements(); } template inline typename RepeatedField::const_iterator RepeatedField::begin() const { - return total_size_ > 0 ? rep()->elements : NULL; + return unsafe_elements(); } template inline typename RepeatedField::const_iterator RepeatedField::cbegin() const { - return total_size_ > 0 ? rep()->elements : NULL; + return unsafe_elements(); } template -inline typename RepeatedField::iterator -RepeatedField::end() { - return total_size_ > 0 ? rep()->elements + current_size_ : NULL; +inline typename RepeatedField::iterator RepeatedField::end() { + return unsafe_elements() + current_size_; } template inline typename RepeatedField::const_iterator RepeatedField::end() const { - return total_size_ > 0 ? rep()->elements + current_size_ : NULL; + return unsafe_elements() + current_size_; } template inline typename RepeatedField::const_iterator RepeatedField::cend() const { - return total_size_ > 0 ? rep()->elements + current_size_ : NULL; + return unsafe_elements() + current_size_; } template @@ -1346,29 +1517,58 @@ inline size_t RepeatedField::SpaceUsedExcludingSelfLong() const { return total_size_ > 0 ? (total_size_ * sizeof(Element) + kRepHeaderSize) : 0; } +namespace internal { +// Returns the new size for a reserved field based on its 'total_size' and the +// requested 'new_size'. The result is clamped to the closed interval: +// [internal::kMinRepeatedFieldAllocationSize, +// std::numeric_limits::max()] +// Requires: +// new_size > total_size && +// (total_size == 0 || +// total_size >= kRepeatedFieldLowerClampLimit) +inline int CalculateReserveSize(int total_size, int new_size) { + if (new_size < kRepeatedFieldLowerClampLimit) { + // Clamp to smallest allowed size. + return kRepeatedFieldLowerClampLimit; + } + if (total_size < kRepeatedFieldUpperClampLimit) { + return std::max(total_size * 2, new_size); + } else { + // Clamp to largest allowed size. + GOOGLE_DCHECK_GT(new_size, kRepeatedFieldUpperClampLimit); + return std::numeric_limits::max(); + } +} +} // namespace internal + // Avoid inlining of Reserve(): new, copy, and delete[] lead to a significant // amount of code bloat. template void RepeatedField::Reserve(int new_size) { if (total_size_ >= new_size) return; Rep* old_rep = total_size_ > 0 ? rep() : NULL; - Arena* arena = GetArenaNoVirtual(); - new_size = std::max(google::protobuf::internal::kMinRepeatedFieldAllocationSize, - std::max(total_size_ * 2, new_size)); + Rep* new_rep; + Arena* arena = GetArena(); + new_size = internal::CalculateReserveSize(total_size_, new_size); GOOGLE_DCHECK_LE( static_cast(new_size), (std::numeric_limits::max() - kRepHeaderSize) / sizeof(Element)) << "Requested size is too large to fit into size_t."; - size_t bytes = kRepHeaderSize + sizeof(Element) * static_cast(new_size); + size_t bytes = + kRepHeaderSize + sizeof(Element) * static_cast(new_size); if (arena == NULL) { - ptr_.rep = static_cast(::operator new(bytes)); + new_rep = static_cast(::operator new(bytes)); } else { - ptr_.rep = reinterpret_cast( - ::google::protobuf::Arena::CreateArray(arena, bytes)); + new_rep = reinterpret_cast(Arena::CreateArray(arena, bytes)); } - ptr_.rep->arena = arena; + new_rep->arena = arena; int old_total_size = total_size_; + // Already known: new_size >= internal::kMinRepeatedFieldAllocationSize + // Maintain invariant: + // total_size_ == 0 || + // total_size_ >= internal::kMinRepeatedFieldAllocationSize total_size_ = new_size; + arena_or_elements_ = new_rep->elements; // Invoke placement-new on newly allocated elements. We shouldn't have to do // this, since Element is supposed to be POD, but a previous version of this // code allocated storage with "new Element[size]" and some code uses @@ -1378,13 +1578,13 @@ void RepeatedField::Reserve(int new_size) { // effect unless its side-effects are required for correctness. // Note that we do this before MoveArray() below because Element's copy // assignment implementation will want an initialized instance first. - Element* e = &rep()->elements[0]; + Element* e = &elements()[0]; Element* limit = e + total_size_; for (; e < limit; e++) { new (e) Element; } if (current_size_ > 0) { - MoveArray(&rep()->elements[0], old_rep->elements, current_size_); + MoveArray(&elements()[0], old_rep->elements, current_size_); } // Likewise, we need to invoke destructors on the old array. @@ -1401,22 +1601,23 @@ inline void RepeatedField::Truncate(int new_size) { } template -inline void RepeatedField::MoveArray( - Element* to, Element* from, int array_size) { +inline void RepeatedField::MoveArray(Element* to, Element* from, + int array_size) { CopyArray(to, from, array_size); } template -inline void RepeatedField::CopyArray( - Element* to, const Element* from, int array_size) { +inline void RepeatedField::CopyArray(Element* to, const Element* from, + int array_size) { internal::ElementCopier()(to, from, array_size); } namespace internal { template -void ElementCopier::operator()( - Element* to, const Element* from, int array_size) { +void ElementCopier::operator()(Element* to, + const Element* from, + int array_size) { std::copy(from, from + array_size, to); } @@ -1435,18 +1636,10 @@ struct ElementCopier { namespace internal { inline RepeatedPtrFieldBase::RepeatedPtrFieldBase() - : arena_(NULL), - current_size_(0), - total_size_(0), - rep_(NULL) { -} + : arena_(NULL), current_size_(0), total_size_(0), rep_(NULL) {} -inline RepeatedPtrFieldBase::RepeatedPtrFieldBase(::google::protobuf::Arena* arena) - : arena_(arena), - current_size_(0), - total_size_(0), - rep_(NULL) { -} +inline RepeatedPtrFieldBase::RepeatedPtrFieldBase(Arena* arena) + : arena_(arena), current_size_(0), total_size_(0), rep_(NULL) {} template void RepeatedPtrFieldBase::Destroy() { @@ -1468,7 +1661,7 @@ void RepeatedPtrFieldBase::Destroy() { template inline void RepeatedPtrFieldBase::Swap(RepeatedPtrFieldBase* other) { - if (other->GetArenaNoVirtual() == GetArenaNoVirtual()) { + if (other->GetArena() == GetArena()) { InternalSwap(other); } else { SwapFallback(other); @@ -1477,39 +1670,48 @@ inline void RepeatedPtrFieldBase::Swap(RepeatedPtrFieldBase* other) { template void RepeatedPtrFieldBase::SwapFallback(RepeatedPtrFieldBase* other) { - GOOGLE_DCHECK(other->GetArenaNoVirtual() != GetArenaNoVirtual()); + GOOGLE_DCHECK(other->GetArena() != GetArena()); // Copy semantics in this case. We try to improve efficiency by placing the - // temporary on |other|'s arena so that messages are copied cross-arena only - // once, not twice. - RepeatedPtrFieldBase temp(other->GetArenaNoVirtual()); + // temporary on |other|'s arena so that messages are copied twice rather than + // three times. + RepeatedPtrFieldBase temp(other->GetArena()); temp.MergeFrom(*this); this->Clear(); this->MergeFrom(*other); - other->Clear(); other->InternalSwap(&temp); temp.Destroy(); // Frees rep_ if `other` had no arena. } -inline bool RepeatedPtrFieldBase::empty() const { - return current_size_ == 0; -} +inline bool RepeatedPtrFieldBase::empty() const { return current_size_ == 0; } -inline int RepeatedPtrFieldBase::size() const { - return current_size_; -} +inline int RepeatedPtrFieldBase::size() const { return current_size_; } template -inline const typename TypeHandler::WeakType& -RepeatedPtrFieldBase::Get(int index) const { +inline const typename TypeHandler::Type& RepeatedPtrFieldBase::Get( + int index) const { GOOGLE_DCHECK_GE(index, 0); GOOGLE_DCHECK_LT(index, current_size_); return *cast(rep_->elements[index]); } template -inline typename TypeHandler::Type* -RepeatedPtrFieldBase::Mutable(int index) { +inline const typename TypeHandler::Type& RepeatedPtrFieldBase::at( + int index) const { + GOOGLE_CHECK_GE(index, 0); + GOOGLE_CHECK_LT(index, current_size_); + return *cast(rep_->elements[index]); +} + +template +inline typename TypeHandler::Type& RepeatedPtrFieldBase::at(int index) { + GOOGLE_CHECK_GE(index, 0); + GOOGLE_CHECK_LT(index, current_size_); + return *cast(rep_->elements[index]); +} + +template +inline typename TypeHandler::Type* RepeatedPtrFieldBase::Mutable(int index) { GOOGLE_DCHECK_GE(index, 0); GOOGLE_DCHECK_LT(index, current_size_); return cast(rep_->elements[index]); @@ -1538,10 +1740,9 @@ inline typename TypeHandler::Type* RepeatedPtrFieldBase::Add( return result; } -template -inline void RepeatedPtrFieldBase::Add( - typename TypeHandler::Type&& value, - std::enable_if*) { +template ::type*> +inline void RepeatedPtrFieldBase::Add(typename TypeHandler::Type&& value) { if (rep_ != NULL && current_size_ < rep_->allocated_size) { *cast(rep_->elements[current_size_++]) = std::move(value); return; @@ -1585,8 +1786,8 @@ template inline void RepeatedPtrFieldBase::MergeFrom(const RepeatedPtrFieldBase& other) { GOOGLE_DCHECK_NE(&other, this); if (other.current_size_ == 0) return; - MergeFromInternal( - other, &RepeatedPtrFieldBase::MergeFromInnerLoop); + MergeFromInternal(other, + &RepeatedPtrFieldBase::MergeFromInnerLoop); } inline void RepeatedPtrFieldBase::MergeFromInternal( @@ -1597,8 +1798,8 @@ inline void RepeatedPtrFieldBase::MergeFromInternal( void** other_elements = other.rep_->elements; void** new_elements = InternalExtend(other_size); int allocated_elems = rep_->allocated_size - current_size_; - (this->*inner_loop)(new_elements, other_elements, - other_size, allocated_elems); + (this->*inner_loop)(new_elements, other_elements, other_size, + allocated_elems); current_size_ += other_size; if (rep_->allocated_size < current_size_) { rep_->allocated_size = current_size_; @@ -1606,25 +1807,26 @@ inline void RepeatedPtrFieldBase::MergeFromInternal( } // Merges other_elems to our_elems. -template -void RepeatedPtrFieldBase::MergeFromInnerLoop( - void** our_elems, void** other_elems, int length, int already_allocated) { +template +void RepeatedPtrFieldBase::MergeFromInnerLoop(void** our_elems, + void** other_elems, int length, + int already_allocated) { // Split into two loops, over ranges [0, allocated) and [allocated, length), // to avoid a branch within the loop. for (int i = 0; i < already_allocated && i < length; i++) { // Already allocated: use existing element. - typename TypeHandler::WeakType* other_elem = - reinterpret_cast(other_elems[i]); - typename TypeHandler::WeakType* new_elem = - reinterpret_cast(our_elems[i]); + typename TypeHandler::Type* other_elem = + reinterpret_cast(other_elems[i]); + typename TypeHandler::Type* new_elem = + reinterpret_cast(our_elems[i]); TypeHandler::Merge(*other_elem, new_elem); } - Arena* arena = GetArenaNoVirtual(); + Arena* arena = GetArena(); for (int i = already_allocated; i < length; i++) { // Not allocated: alloc a new element first, then merge it. - typename TypeHandler::WeakType* other_elem = - reinterpret_cast(other_elems[i]); - typename TypeHandler::WeakType* new_elem = + typename TypeHandler::Type* other_elem = + reinterpret_cast(other_elems[i]); + typename TypeHandler::Type* new_elem = TypeHandler::NewFromPrototype(other_elem, arena); TypeHandler::Merge(*other_elem, new_elem); our_elems[i] = new_elem; @@ -1638,9 +1840,7 @@ inline void RepeatedPtrFieldBase::CopyFrom(const RepeatedPtrFieldBase& other) { RepeatedPtrFieldBase::MergeFrom(other); } -inline int RepeatedPtrFieldBase::Capacity() const { - return total_size_; -} +inline int RepeatedPtrFieldBase::Capacity() const { return total_size_; } inline void* const* RepeatedPtrFieldBase::raw_data() const { return rep_ ? rep_->elements : NULL; @@ -1658,8 +1858,8 @@ inline typename TypeHandler::Type** RepeatedPtrFieldBase::mutable_data() { } template -inline const typename TypeHandler::Type* const* -RepeatedPtrFieldBase::data() const { +inline const typename TypeHandler::Type* const* RepeatedPtrFieldBase::data() + const { // TODO(kenton): Breaks C++ aliasing rules. We should probably remove this // method entirely. return reinterpret_cast(raw_data()); @@ -1675,8 +1875,8 @@ inline size_t RepeatedPtrFieldBase::SpaceUsedExcludingSelfLong() const { size_t allocated_bytes = static_cast(total_size_) * sizeof(void*); if (rep_ != NULL) { for (int i = 0; i < rep_->allocated_size; ++i) { - allocated_bytes += TypeHandler::SpaceUsedLong( - *cast(rep_->elements[i])); + allocated_bytes += + TypeHandler::SpaceUsedLong(*cast(rep_->elements[i])); } allocated_bytes += kRepHeaderSize; } @@ -1695,13 +1895,11 @@ inline typename TypeHandler::Type* RepeatedPtrFieldBase::AddFromCleared() { // AddAllocated version that implements arena-safe copying behavior. template void RepeatedPtrFieldBase::AddAllocatedInternal( - typename TypeHandler::Type* value, - std::true_type) { - Arena* element_arena = reinterpret_cast( - TypeHandler::GetMaybeArenaPointer(value)); - Arena* arena = GetArenaNoVirtual(); - if (arena == element_arena && rep_ && - rep_->allocated_size < total_size_) { + typename TypeHandler::Type* value, std::true_type) { + Arena* element_arena = + reinterpret_cast(TypeHandler::GetMaybeArenaPointer(value)); + Arena* arena = GetArena(); + if (arena == element_arena && rep_ && rep_->allocated_size < total_size_) { // Fast path: underlying arena representation (tagged pointer) is equal to // our arena pointer, and we can add to array without resizing it (at least // one slot that is not allocated). @@ -1715,13 +1913,13 @@ void RepeatedPtrFieldBase::AddAllocatedInternal( current_size_ = current_size_ + 1; rep_->allocated_size = rep_->allocated_size + 1; } else { - AddAllocatedSlowWithCopy( - value, TypeHandler::GetArena(value), arena); + AddAllocatedSlowWithCopy(value, TypeHandler::GetArena(value), + arena); } } // Slowpath handles all cases, copying if necessary. -template +template void RepeatedPtrFieldBase::AddAllocatedSlowWithCopy( // Pass value_arena and my_arena to avoid duplicate virtual call (value) or // load (mine). @@ -1745,9 +1943,8 @@ void RepeatedPtrFieldBase::AddAllocatedSlowWithCopy( // AddAllocated version that does not implement arena-safe copying behavior. template void RepeatedPtrFieldBase::AddAllocatedInternal( - typename TypeHandler::Type* value, - std::false_type) { - if (rep_ && rep_->allocated_size < total_size_) { + typename TypeHandler::Type* value, std::false_type) { + if (rep_ && rep_->allocated_size < total_size_) { // Fast path: underlying arena representation (tagged pointer) is equal to // our arena pointer, and we can add to array without resizing it (at least // one slot that is not allocated). @@ -1778,8 +1975,8 @@ void RepeatedPtrFieldBase::UnsafeArenaAddAllocated( // cleared objects awaiting reuse. We don't want to grow the array in this // case because otherwise a loop calling AddAllocated() followed by Clear() // would leak memory. - TypeHandler::Delete( - cast(rep_->elements[current_size_]), arena_); + TypeHandler::Delete(cast(rep_->elements[current_size_]), + arena_); } else if (current_size_ < rep_->allocated_size) { // We have some cleared objects. We don't care about their order, so we // can just move the first one to the end to make space. @@ -1795,12 +1992,12 @@ void RepeatedPtrFieldBase::UnsafeArenaAddAllocated( // ReleaseLast() for types that implement merge/copy behavior. template -inline typename TypeHandler::Type* -RepeatedPtrFieldBase::ReleaseLastInternal(std::true_type) { +inline typename TypeHandler::Type* RepeatedPtrFieldBase::ReleaseLastInternal( + std::true_type) { // First, release an element. typename TypeHandler::Type* result = UnsafeArenaReleaseLast(); // Now perform a copy if we're on an arena. - Arena* arena = GetArenaNoVirtual(); + Arena* arena = GetArena(); if (arena == NULL) { return result; } else { @@ -1816,9 +2013,9 @@ RepeatedPtrFieldBase::ReleaseLastInternal(std::true_type) { // an arena, since the user really should implement the copy operation in this // case. template -inline typename TypeHandler::Type* -RepeatedPtrFieldBase::ReleaseLastInternal(std::false_type) { - GOOGLE_DCHECK(GetArenaNoVirtual() == NULL) +inline typename TypeHandler::Type* RepeatedPtrFieldBase::ReleaseLastInternal( + std::false_type) { + GOOGLE_DCHECK(GetArena() == NULL) << "ReleaseLast() called on a RepeatedPtrField that is on an arena, " << "with a type that does not implement MergeFrom. This is unsafe; " << "please implement MergeFrom for your type."; @@ -1827,7 +2024,7 @@ RepeatedPtrFieldBase::ReleaseLastInternal(std::false_type) { template inline typename TypeHandler::Type* - RepeatedPtrFieldBase::UnsafeArenaReleaseLast() { +RepeatedPtrFieldBase::UnsafeArenaReleaseLast() { GOOGLE_DCHECK_GT(current_size_, 0); typename TypeHandler::Type* result = cast(rep_->elements[--current_size_]); @@ -1847,7 +2044,7 @@ inline int RepeatedPtrFieldBase::ClearedCount() const { template inline void RepeatedPtrFieldBase::AddCleared( typename TypeHandler::Type* value) { - GOOGLE_DCHECK(GetArenaNoVirtual() == NULL) + GOOGLE_DCHECK(GetArena() == NULL) << "AddCleared() can only be used on a RepeatedPtrField not on an arena."; GOOGLE_DCHECK(TypeHandler::GetArena(value) == NULL) << "AddCleared() can only accept values not on an arena."; @@ -1859,10 +2056,10 @@ inline void RepeatedPtrFieldBase::AddCleared( template inline typename TypeHandler::Type* RepeatedPtrFieldBase::ReleaseCleared() { - GOOGLE_DCHECK(GetArenaNoVirtual() == NULL) + GOOGLE_DCHECK(GetArena() == NULL) << "ReleaseCleared() can only be used on a RepeatedPtrField not on " << "an arena."; - GOOGLE_DCHECK(GetArenaNoVirtual() == NULL); + GOOGLE_DCHECK(GetArena() == NULL); GOOGLE_DCHECK(rep_ != NULL); GOOGLE_DCHECK_GT(rep_->allocated_size, current_size_); return cast(rep_->elements[--rep_->allocated_size]); @@ -1874,33 +2071,30 @@ inline typename TypeHandler::Type* RepeatedPtrFieldBase::ReleaseCleared() { template class RepeatedPtrField::TypeHandler - : public internal::GenericTypeHandler { -}; + : public internal::GenericTypeHandler {}; template <> -class RepeatedPtrField::TypeHandler - : public internal::StringTypeHandler { -}; +class RepeatedPtrField::TypeHandler + : public internal::StringTypeHandler {}; template -inline RepeatedPtrField::RepeatedPtrField() - : RepeatedPtrFieldBase() {} +inline RepeatedPtrField::RepeatedPtrField() : RepeatedPtrFieldBase() {} template -inline RepeatedPtrField::RepeatedPtrField(::google::protobuf::Arena* arena) : - RepeatedPtrFieldBase(arena) {} +inline RepeatedPtrField::RepeatedPtrField(Arena* arena) + : RepeatedPtrFieldBase(arena) {} template inline RepeatedPtrField::RepeatedPtrField( const RepeatedPtrField& other) - : RepeatedPtrFieldBase() { + : RepeatedPtrFieldBase() { MergeFrom(other); } template template -inline RepeatedPtrField::RepeatedPtrField( - Iter begin, const Iter& end) { +inline RepeatedPtrField::RepeatedPtrField(Iter begin, + const Iter& end) { int reserve = internal::CalculateReserve(begin, end); if (reserve != -1) { Reserve(reserve); @@ -1918,8 +2112,7 @@ RepeatedPtrField::~RepeatedPtrField() { template inline RepeatedPtrField& RepeatedPtrField::operator=( const RepeatedPtrField& other) { - if (this != &other) - CopyFrom(other); + if (this != &other) CopyFrom(other); return *this; } @@ -1928,8 +2121,9 @@ inline RepeatedPtrField::RepeatedPtrField( RepeatedPtrField&& other) noexcept : RepeatedPtrField() { // We don't just call Swap(&other) here because it would perform 3 copies if - // the two fields are on different arenas. - if (other.GetArenaNoVirtual()) { + // other is on an arena. This field can't be on an arena because arena + // construction always uses the Arena* accepting constructor. + if (other.GetArena()) { CopyFrom(other); } else { InternalSwap(&other); @@ -1942,7 +2136,7 @@ inline RepeatedPtrField& RepeatedPtrField::operator=( // We don't just call Swap(&other) here because it would perform 3 copies if // the two fields are on different arenas. if (this != &other) { - if (this->GetArenaNoVirtual() != other.GetArenaNoVirtual()) { + if (this->GetArena() != other.GetArena()) { CopyFrom(other); } else { InternalSwap(&other); @@ -1966,6 +2160,16 @@ inline const Element& RepeatedPtrField::Get(int index) const { return RepeatedPtrFieldBase::Get(index); } +template +inline const Element& RepeatedPtrField::at(int index) const { + return RepeatedPtrFieldBase::at(index); +} + +template +inline Element& RepeatedPtrField::at(int index) { + return RepeatedPtrFieldBase::at(index); +} + template inline Element* RepeatedPtrField::Mutable(int index) { @@ -1999,8 +2203,8 @@ inline void RepeatedPtrField::DeleteSubrange(int start, int num) { } template -inline void RepeatedPtrField::ExtractSubrange( - int start, int num, Element** elements) { +inline void RepeatedPtrField::ExtractSubrange(int start, int num, + Element** elements) { typename internal::TypeImplementsMergeBehavior< typename TypeHandler::Type>::type t; ExtractSubrangeInternal(start, num, elements, t); @@ -2018,12 +2222,12 @@ inline void RepeatedPtrField::ExtractSubrangeInternal( if (num > 0) { // Save the values of the removed elements if requested. if (elements != NULL) { - if (GetArenaNoVirtual() != NULL) { + if (GetArena() != NULL) { // If we're on an arena, we perform a copy for each element so that the // returned elements are heap-allocated. for (int i = 0; i < num; ++i) { - Element* element = RepeatedPtrFieldBase:: - Mutable(i + start); + Element* element = + RepeatedPtrFieldBase::Mutable(i + start); typename TypeHandler::Type* new_value = TypeHandler::NewFromPrototype(element, NULL); TypeHandler::Merge(*element, new_value); @@ -2041,14 +2245,14 @@ inline void RepeatedPtrField::ExtractSubrangeInternal( // ExtractSubrange() implementation for types that do not implement merge/copy // behavior. -template +template inline void RepeatedPtrField::ExtractSubrangeInternal( int start, int num, Element** elements, std::false_type) { // This case is identical to UnsafeArenaExtractSubrange(). However, since // ExtractSubrange() must return heap-allocated objects by contract, and we // cannot fulfill this contract if we are an on arena, we must GOOGLE_DCHECK() that // we are not on an arena. - GOOGLE_DCHECK(GetArenaNoVirtual() == NULL) + GOOGLE_DCHECK(GetArena() == NULL) << "ExtractSubrange() when arena is non-NULL is only supported when " << "the Element type supplies a MergeFrom() operation to make copies."; UnsafeArenaExtractSubrange(start, num, elements); @@ -2084,8 +2288,7 @@ inline void RepeatedPtrField::MergeFrom( } template -inline void RepeatedPtrField::CopyFrom( - const RepeatedPtrField& other) { +inline void RepeatedPtrField::CopyFrom(const RepeatedPtrField& other) { RepeatedPtrFieldBase::CopyFrom(other); } @@ -2116,16 +2319,14 @@ inline const Element* const* RepeatedPtrField::data() const { template inline void RepeatedPtrField::Swap(RepeatedPtrField* other) { - if (this == other) - return; + if (this == other) return; RepeatedPtrFieldBase::Swap(other); } template inline void RepeatedPtrField::UnsafeArenaSwap( RepeatedPtrField* other) { - if (this == other) - return; + if (this == other) return; RepeatedPtrFieldBase::InternalSwap(other); } @@ -2135,8 +2336,8 @@ inline void RepeatedPtrField::SwapElements(int index1, int index2) { } template -inline Arena* RepeatedPtrField::GetArenaNoVirtual() const { - return RepeatedPtrFieldBase::GetArenaNoVirtual(); +inline Arena* RepeatedPtrField::GetArena() const { + return RepeatedPtrFieldBase::GetArena(); } template @@ -2204,31 +2405,22 @@ namespace internal { // // This code based on net/proto/proto-array-internal.h by Jeffrey Yasskin // (jyasskin@google.com). -template -class RepeatedPtrIterator - : public std::iterator< - std::random_access_iterator_tag, Element> { +template +class RepeatedPtrIterator { public: - typedef RepeatedPtrIterator iterator; - typedef std::iterator< - std::random_access_iterator_tag, Element> superclass; - - // Shadow the value_type in std::iterator<> because const_iterator::value_type - // needs to be T, not const T. - typedef typename std::remove_const::type value_type; - - // Let the compiler know that these are type names, so we don't have to - // write "typename" in front of them everywhere. - typedef typename superclass::reference reference; - typedef typename superclass::pointer pointer; - typedef typename superclass::difference_type difference_type; + using iterator = RepeatedPtrIterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = typename std::remove_const::type; + using difference_type = std::ptrdiff_t; + using pointer = Element*; + using reference = Element&; RepeatedPtrIterator() : it_(NULL) {} explicit RepeatedPtrIterator(void* const* it) : it_(it) {} // Allow "upcasting" from RepeatedPtrIterator to // RepeatedPtrIterator. - template + template RepeatedPtrIterator(const RepeatedPtrIterator& other) : it_(other.it_) { // Force a compiler error if the other type is not convertible to ours. @@ -2239,13 +2431,19 @@ class RepeatedPtrIterator // dereferenceable reference operator*() const { return *reinterpret_cast(*it_); } - pointer operator->() const { return &(operator*()); } + pointer operator->() const { return &(operator*()); } // {inc,dec}rementable - iterator& operator++() { ++it_; return *this; } - iterator operator++(int) { return iterator(it_++); } - iterator& operator--() { --it_; return *this; } - iterator operator--(int) { return iterator(it_--); } + iterator& operator++() { + ++it_; + return *this; + } + iterator operator++(int) { return iterator(it_++); } + iterator& operator--() { + --it_; + return *this; + } + iterator operator--(int) { return iterator(it_--); } // equality_comparable bool operator==(const iterator& x) const { return it_ == x.it_; } @@ -2286,7 +2484,7 @@ class RepeatedPtrIterator difference_type operator-(const iterator& x) const { return it_ - x.it_; } private: - template + template friend class RepeatedPtrIterator; // The internal iterator. @@ -2301,34 +2499,33 @@ class RepeatedPtrIterator // referenced by the iterator. It should either be "void *" for a mutable // iterator, or "const void* const" for a constant iterator. template -class RepeatedPtrOverPtrsIterator - : public std::iterator { +class RepeatedPtrOverPtrsIterator { public: - typedef RepeatedPtrOverPtrsIterator iterator; - typedef std::iterator superclass; - - // Shadow the value_type in std::iterator<> because const_iterator::value_type - // needs to be T, not const T. - typedef typename std::remove_const::type value_type; - - // Let the compiler know that these are type names, so we don't have to - // write "typename" in front of them everywhere. - typedef typename superclass::reference reference; - typedef typename superclass::pointer pointer; - typedef typename superclass::difference_type difference_type; + using iterator = RepeatedPtrOverPtrsIterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = typename std::remove_const::type; + using difference_type = std::ptrdiff_t; + using pointer = Element*; + using reference = Element&; RepeatedPtrOverPtrsIterator() : it_(NULL) {} explicit RepeatedPtrOverPtrsIterator(VoidPtr* it) : it_(it) {} // dereferenceable reference operator*() const { return *reinterpret_cast(it_); } - pointer operator->() const { return &(operator*()); } + pointer operator->() const { return &(operator*()); } // {inc,dec}rementable - iterator& operator++() { ++it_; return *this; } - iterator operator++(int) { return iterator(it_++); } - iterator& operator--() { --it_; return *this; } - iterator operator--(int) { return iterator(it_--); } + iterator& operator++() { + ++it_; + return *this; + } + iterator operator++(int) { return iterator(it_++); } + iterator& operator--() { + --it_; + return *this; + } + iterator operator--(int) { return iterator(it_--); } // equality_comparable bool operator==(const iterator& x) const { return it_ == x.it_; } @@ -2369,7 +2566,7 @@ class RepeatedPtrOverPtrsIterator difference_type operator-(const iterator& x) const { return it_ - x.it_; } private: - template + template friend class RepeatedPtrIterator; // The internal iterator. @@ -2378,11 +2575,17 @@ class RepeatedPtrOverPtrsIterator void RepeatedPtrFieldBase::InternalSwap(RepeatedPtrFieldBase* other) { GOOGLE_DCHECK(this != other); - GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); + GOOGLE_DCHECK(GetArena() == other->GetArena()); - std::swap(rep_, other->rep_); - std::swap(current_size_, other->current_size_); - std::swap(total_size_, other->total_size_); + // Swap all fields at once. + static_assert(std::is_standard_layout::value, + "offsetof() requires standard layout before c++17"); + internal::memswaprep_) - + offsetof(RepeatedPtrFieldBase, current_size_)>( + reinterpret_cast(this) + + offsetof(RepeatedPtrFieldBase, current_size_), + reinterpret_cast(other) + + offsetof(RepeatedPtrFieldBase, current_size_)); } } // namespace internal @@ -2440,35 +2643,30 @@ RepeatedPtrField::pointer_end() const { const_cast(raw_data() + size())); } - // Iterators and helper functions that follow the spirit of the STL // std::back_insert_iterator and std::back_inserter but are tailor-made // for RepeatedField and RepeatedPtrField. Typical usage would be: // // std::copy(some_sequence.begin(), some_sequence.end(), -// google::protobuf::RepeatedFieldBackInserter(proto.mutable_sequence())); +// RepeatedFieldBackInserter(proto.mutable_sequence())); // // Ported by johannes from util/gtl/proto-array-iterators.h namespace internal { // A back inserter for RepeatedField objects. -template class RepeatedFieldBackInsertIterator +template +class RepeatedFieldBackInsertIterator : public std::iterator { public: explicit RepeatedFieldBackInsertIterator( RepeatedField* const mutable_field) - : field_(mutable_field) { - } + : field_(mutable_field) {} RepeatedFieldBackInsertIterator& operator=(const T& value) { field_->Add(value); return *this; } - RepeatedFieldBackInsertIterator& operator*() { - return *this; - } - RepeatedFieldBackInsertIterator& operator++() { - return *this; - } + RepeatedFieldBackInsertIterator& operator*() { return *this; } + RepeatedFieldBackInsertIterator& operator++() { return *this; } RepeatedFieldBackInsertIterator& operator++(int /* unused */) { return *this; } @@ -2478,13 +2676,12 @@ template class RepeatedFieldBackInsertIterator }; // A back inserter for RepeatedPtrField objects. -template class RepeatedPtrFieldBackInsertIterator +template +class RepeatedPtrFieldBackInsertIterator : public std::iterator { public: - RepeatedPtrFieldBackInsertIterator( - RepeatedPtrField* const mutable_field) - : field_(mutable_field) { - } + RepeatedPtrFieldBackInsertIterator(RepeatedPtrField* const mutable_field) + : field_(mutable_field) {} RepeatedPtrFieldBackInsertIterator& operator=(const T& value) { *field_->Add() = value; return *this; @@ -2498,12 +2695,8 @@ template class RepeatedPtrFieldBackInsertIterator *field_->Add() = std::move(value); return *this; } - RepeatedPtrFieldBackInsertIterator& operator*() { - return *this; - } - RepeatedPtrFieldBackInsertIterator& operator++() { - return *this; - } + RepeatedPtrFieldBackInsertIterator& operator*() { return *this; } + RepeatedPtrFieldBackInsertIterator& operator++() { return *this; } RepeatedPtrFieldBackInsertIterator& operator++(int /* unused */) { return *this; } @@ -2514,26 +2707,21 @@ template class RepeatedPtrFieldBackInsertIterator // A back inserter for RepeatedPtrFields that inserts by transferring ownership // of a pointer. -template class AllocatedRepeatedPtrFieldBackInsertIterator +template +class AllocatedRepeatedPtrFieldBackInsertIterator : public std::iterator { public: explicit AllocatedRepeatedPtrFieldBackInsertIterator( RepeatedPtrField* const mutable_field) - : field_(mutable_field) { - } + : field_(mutable_field) {} AllocatedRepeatedPtrFieldBackInsertIterator& operator=( T* const ptr_to_value) { field_->AddAllocated(ptr_to_value); return *this; } - AllocatedRepeatedPtrFieldBackInsertIterator& operator*() { - return *this; - } - AllocatedRepeatedPtrFieldBackInsertIterator& operator++() { - return *this; - } - AllocatedRepeatedPtrFieldBackInsertIterator& operator++( - int /* unused */) { + AllocatedRepeatedPtrFieldBackInsertIterator& operator*() { return *this; } + AllocatedRepeatedPtrFieldBackInsertIterator& operator++() { return *this; } + AllocatedRepeatedPtrFieldBackInsertIterator& operator++(int /* unused */) { return *this; } @@ -2543,16 +2731,15 @@ template class AllocatedRepeatedPtrFieldBackInsertIterator // Almost identical to AllocatedRepeatedPtrFieldBackInsertIterator. This one // uses the UnsafeArenaAddAllocated instead. -template +template class UnsafeArenaAllocatedRepeatedPtrFieldBackInsertIterator : public std::iterator { public: explicit UnsafeArenaAllocatedRepeatedPtrFieldBackInsertIterator( - ::google::protobuf::RepeatedPtrField* const mutable_field) - : field_(mutable_field) { - } + RepeatedPtrField* const mutable_field) + : field_(mutable_field) {} UnsafeArenaAllocatedRepeatedPtrFieldBackInsertIterator& operator=( - T const* const ptr_to_value) { + T const* const ptr_to_value) { field_->UnsafeArenaAddAllocated(const_cast(ptr_to_value)); return *this; } @@ -2568,37 +2755,41 @@ class UnsafeArenaAllocatedRepeatedPtrFieldBackInsertIterator } private: - ::google::protobuf::RepeatedPtrField* field_; + RepeatedPtrField* field_; }; } // namespace internal // Provides a back insert iterator for RepeatedField instances, // similar to std::back_inserter(). -template internal::RepeatedFieldBackInsertIterator -RepeatedFieldBackInserter(RepeatedField* const mutable_field) { +template +internal::RepeatedFieldBackInsertIterator RepeatedFieldBackInserter( + RepeatedField* const mutable_field) { return internal::RepeatedFieldBackInsertIterator(mutable_field); } // Provides a back insert iterator for RepeatedPtrField instances, // similar to std::back_inserter(). -template internal::RepeatedPtrFieldBackInsertIterator -RepeatedPtrFieldBackInserter(RepeatedPtrField* const mutable_field) { +template +internal::RepeatedPtrFieldBackInsertIterator RepeatedPtrFieldBackInserter( + RepeatedPtrField* const mutable_field) { return internal::RepeatedPtrFieldBackInsertIterator(mutable_field); } // Special back insert iterator for RepeatedPtrField instances, just in // case someone wants to write generic template code that can access both // RepeatedFields and RepeatedPtrFields using a common name. -template internal::RepeatedPtrFieldBackInsertIterator -RepeatedFieldBackInserter(RepeatedPtrField* const mutable_field) { +template +internal::RepeatedPtrFieldBackInsertIterator RepeatedFieldBackInserter( + RepeatedPtrField* const mutable_field) { return internal::RepeatedPtrFieldBackInsertIterator(mutable_field); } // Provides a back insert iterator for RepeatedPtrField instances // similar to std::back_inserter() which transfers the ownership while // copying elements. -template internal::AllocatedRepeatedPtrFieldBackInsertIterator +template +internal::AllocatedRepeatedPtrFieldBackInsertIterator AllocatedRepeatedPtrFieldBackInserter( RepeatedPtrField* const mutable_field) { return internal::AllocatedRepeatedPtrFieldBackInsertIterator( @@ -2616,15 +2807,28 @@ AllocatedRepeatedPtrFieldBackInserter( // If you put temp_field on the arena this fails, because the ownership // transfers to the arena at the "AddAllocated" call and is not released anymore // causing a double delete. Using UnsafeArenaAddAllocated prevents this. -template +template internal::UnsafeArenaAllocatedRepeatedPtrFieldBackInsertIterator UnsafeArenaAllocatedRepeatedPtrFieldBackInserter( - ::google::protobuf::RepeatedPtrField* const mutable_field) { + RepeatedPtrField* const mutable_field) { return internal::UnsafeArenaAllocatedRepeatedPtrFieldBackInsertIterator( mutable_field); } -} // namespace protobuf +// Extern declarations of common instantiations to reduce library bloat. +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField; +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField; +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField; +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField; +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField; +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField; +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField; +extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE + RepeatedPtrField; +} // namespace protobuf } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_REPEATED_FIELD_H__ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h b/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h index 86510d14..0193301a 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h @@ -56,8 +56,9 @@ #include #include +#include + class CordByteSink; -class MemBlock; namespace google { namespace protobuf { @@ -74,7 +75,7 @@ namespace strings { // sink->Append(my_data.data(), my_data.size()); // sink->Flush(); // -class LIBPROTOBUF_EXPORT ByteSink { +class PROTOBUF_EXPORT ByteSink { public: ByteSink() {} virtual ~ByteSink() {} @@ -82,7 +83,7 @@ class LIBPROTOBUF_EXPORT ByteSink { // Appends the "n" bytes starting at "bytes". virtual void Append(const char* bytes, size_t n) = 0; - // Flushes internal buffers. The default implemenation does nothing. ByteSink + // Flushes internal buffers. The default implementation does nothing. ByteSink // subclasses may use internal buffers that require calling Flush() at the end // of the stream. virtual void Flush(); @@ -103,7 +104,7 @@ class LIBPROTOBUF_EXPORT ByteSink { // source->Skip(data.length()); // } // -class LIBPROTOBUF_EXPORT ByteSource { +class PROTOBUF_EXPORT ByteSource { public: ByteSource() {} virtual ~ByteSource() {} @@ -159,10 +160,10 @@ class LIBPROTOBUF_EXPORT ByteSource { // sink.Append("hi", 2); // OK // sink.Append(data, 100); // WOOPS! Overflows buf[10]. // -class LIBPROTOBUF_EXPORT UncheckedArrayByteSink : public ByteSink { +class PROTOBUF_EXPORT UncheckedArrayByteSink : public ByteSink { public: explicit UncheckedArrayByteSink(char* dest) : dest_(dest) {} - virtual void Append(const char* data, size_t n); + virtual void Append(const char* data, size_t n) override; // Returns the current output pointer so that a caller can see how many bytes // were produced. @@ -187,10 +188,10 @@ class LIBPROTOBUF_EXPORT UncheckedArrayByteSink : public ByteSink { // sink.Append("hi", 2); // OK // sink.Append(data, 100); // Will only write 8 more bytes // -class LIBPROTOBUF_EXPORT CheckedArrayByteSink : public ByteSink { +class PROTOBUF_EXPORT CheckedArrayByteSink : public ByteSink { public: CheckedArrayByteSink(char* outbuf, size_t capacity); - virtual void Append(const char* bytes, size_t n); + virtual void Append(const char* bytes, size_t n) override; // Returns the number of bytes actually written to the sink. size_t NumberOfBytesWritten() const { return size_; } @@ -223,11 +224,11 @@ class LIBPROTOBUF_EXPORT CheckedArrayByteSink : public ByteSink { // const char* buf = sink.GetBuffer(); // Ownership transferred // delete[] buf; // -class LIBPROTOBUF_EXPORT GrowingArrayByteSink : public strings::ByteSink { +class PROTOBUF_EXPORT GrowingArrayByteSink : public strings::ByteSink { public: explicit GrowingArrayByteSink(size_t estimated_size); virtual ~GrowingArrayByteSink(); - virtual void Append(const char* bytes, size_t n); + virtual void Append(const char* bytes, size_t n) override; // Returns the allocated buffer, and sets nbytes to its size. The caller takes // ownership of the buffer and must delete it with delete[]. @@ -253,10 +254,10 @@ class LIBPROTOBUF_EXPORT GrowingArrayByteSink : public strings::ByteSink { // sink.Append("World", 5); // assert(dest == "Hello World"); // -class LIBPROTOBUF_EXPORT StringByteSink : public ByteSink { +class PROTOBUF_EXPORT StringByteSink : public ByteSink { public: explicit StringByteSink(string* dest) : dest_(dest) {} - virtual void Append(const char* data, size_t n); + virtual void Append(const char* data, size_t n) override; private: string* dest_; @@ -270,10 +271,10 @@ class LIBPROTOBUF_EXPORT StringByteSink : public ByteSink { // NullByteSink sink; // sink.Append(data, data.size()); // All data ignored. // -class LIBPROTOBUF_EXPORT NullByteSink : public ByteSink { +class PROTOBUF_EXPORT NullByteSink : public ByteSink { public: NullByteSink() {} - virtual void Append(const char *data, size_t n) {} + void Append(const char* /*data*/, size_t /*n*/) override {} private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(NullByteSink); @@ -292,13 +293,13 @@ class LIBPROTOBUF_EXPORT NullByteSink : public ByteSink { // assert(source.Available() == 5); // assert(source.Peek() == "Hello"); // -class LIBPROTOBUF_EXPORT ArrayByteSource : public ByteSource { +class PROTOBUF_EXPORT ArrayByteSource : public ByteSource { public: explicit ArrayByteSource(StringPiece s) : input_(s) {} - virtual size_t Available() const; - virtual StringPiece Peek(); - virtual void Skip(size_t n); + virtual size_t Available() const override; + virtual StringPiece Peek() override; + virtual void Skip(size_t n) override; private: StringPiece input_; @@ -323,18 +324,18 @@ class LIBPROTOBUF_EXPORT ArrayByteSource : public ByteSource { // assert(limit.Available() == 5); // assert(limit.Peek() == "Hello"); // -class LIBPROTOBUF_EXPORT LimitByteSource : public ByteSource { +class PROTOBUF_EXPORT LimitByteSource : public ByteSource { public: // Returns at most "limit" bytes from "source". LimitByteSource(ByteSource* source, size_t limit); - virtual size_t Available() const; - virtual StringPiece Peek(); - virtual void Skip(size_t n); + virtual size_t Available() const override; + virtual StringPiece Peek() override; + virtual void Skip(size_t n) override; // We override CopyTo so that we can forward to the underlying source, in // case it has an efficient implementation of CopyTo. - virtual void CopyTo(ByteSink* sink, size_t n); + virtual void CopyTo(ByteSink* sink, size_t n) override; private: ByteSource* source_; @@ -345,4 +346,6 @@ class LIBPROTOBUF_EXPORT LimitByteSource : public ByteSource { } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_BYTESTREAM_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/callback.h b/third_party/protobuf-lite/google/protobuf/stubs/callback.h index 6888f136..b7a3a829 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/callback.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/callback.h @@ -5,6 +5,8 @@ #include +#include + // =================================================================== // emulates google3/base/callback.h @@ -66,9 +68,9 @@ namespace protobuf { // Also note that the arguments cannot be references: // void Foo(const string& s); // string my_str; -// NewCallback(&Foo, my_str); // WON'T WORK: Can't use referecnes. +// NewCallback(&Foo, my_str); // WON'T WORK: Can't use references. // However, correctly-typed pointers will work just fine. -class LIBPROTOBUF_EXPORT Closure { +class PROTOBUF_EXPORT Closure { public: Closure() {} virtual ~Closure(); @@ -91,8 +93,8 @@ class ResultCallback { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ResultCallback); }; -template -class LIBPROTOBUF_EXPORT ResultCallback1 { +template +class PROTOBUF_EXPORT ResultCallback1 { public: ResultCallback1() {} virtual ~ResultCallback1() {} @@ -103,8 +105,8 @@ class LIBPROTOBUF_EXPORT ResultCallback1 { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ResultCallback1); }; -template -class LIBPROTOBUF_EXPORT ResultCallback2 { +template +class PROTOBUF_EXPORT ResultCallback2 { public: ResultCallback2() {} virtual ~ResultCallback2() {} @@ -117,7 +119,7 @@ class LIBPROTOBUF_EXPORT ResultCallback2 { namespace internal { -class LIBPROTOBUF_EXPORT FunctionClosure0 : public Closure { +class PROTOBUF_EXPORT FunctionClosure0 : public Closure { public: typedef void (*FunctionType)(); @@ -125,7 +127,7 @@ class LIBPROTOBUF_EXPORT FunctionClosure0 : public Closure { : function_(function), self_deleting_(self_deleting) {} ~FunctionClosure0(); - void Run() { + void Run() override { bool needs_delete = self_deleting_; // read in case callback deletes function_(); if (needs_delete) delete this; @@ -145,7 +147,7 @@ class MethodClosure0 : public Closure { : object_(object), method_(method), self_deleting_(self_deleting) {} ~MethodClosure0() {} - void Run() { + void Run() override { bool needs_delete = self_deleting_; // read in case callback deletes (object_->*method_)(); if (needs_delete) delete this; @@ -168,7 +170,7 @@ class FunctionClosure1 : public Closure { arg1_(arg1) {} ~FunctionClosure1() {} - void Run() { + void Run() override { bool needs_delete = self_deleting_; // read in case callback deletes function_(arg1_); if (needs_delete) delete this; @@ -191,7 +193,7 @@ class MethodClosure1 : public Closure { arg1_(arg1) {} ~MethodClosure1() {} - void Run() { + void Run() override { bool needs_delete = self_deleting_; // read in case callback deletes (object_->*method_)(arg1_); if (needs_delete) delete this; @@ -215,7 +217,7 @@ class FunctionClosure2 : public Closure { arg1_(arg1), arg2_(arg2) {} ~FunctionClosure2() {} - void Run() { + void Run() override { bool needs_delete = self_deleting_; // read in case callback deletes function_(arg1_, arg2_); if (needs_delete) delete this; @@ -239,7 +241,7 @@ class MethodClosure2 : public Closure { arg1_(arg1), arg2_(arg2) {} ~MethodClosure2() {} - void Run() { + void Run() override { bool needs_delete = self_deleting_; // read in case callback deletes (object_->*method_)(arg1_, arg2_); if (needs_delete) delete this; @@ -262,7 +264,7 @@ class FunctionResultCallback_0_0 : public ResultCallback { : function_(function), self_deleting_(self_deleting) {} ~FunctionResultCallback_0_0() {} - R Run() { + R Run() override { bool needs_delete = self_deleting_; // read in case callback deletes R result = function_(); if (needs_delete) delete this; @@ -284,7 +286,7 @@ class FunctionResultCallback_1_0 : public ResultCallback { : function_(function), self_deleting_(self_deleting), p1_(p1) {} ~FunctionResultCallback_1_0() {} - R Run() { + R Run() override { bool needs_delete = self_deleting_; // read in case callback deletes R result = function_(p1_); if (needs_delete) delete this; @@ -306,7 +308,7 @@ class FunctionResultCallback_0_1 : public ResultCallback1 { : function_(function), self_deleting_(self_deleting) {} ~FunctionResultCallback_0_1() {} - R Run(Arg1 a1) { + R Run(Arg1 a1) override { bool needs_delete = self_deleting_; // read in case callback deletes R result = function_(a1); if (needs_delete) delete this; @@ -328,7 +330,7 @@ class FunctionResultCallback_1_1 : public ResultCallback1 { : function_(function), self_deleting_(self_deleting), p1_(p1) {} ~FunctionResultCallback_1_1() {} - R Run(A1 a1) { + R Run(A1 a1) override { bool needs_delete = self_deleting_; // read in case callback deletes R result = function_(p1_, a1); if (needs_delete) delete this; @@ -371,12 +373,12 @@ class MethodResultCallback_0_0 : public ResultCallback { }; template -class MethodResultCallback_5_2 : public ResultCallback2 { + typename P4, typename P5, typename P6, typename A1, typename A2> +class MethodResultCallback_6_2 : public ResultCallback2 { public: - typedef R (T::*MethodType)(P1, P2, P3, P4, P5, A1, A2); - MethodResultCallback_5_2(T* object, MethodType method, bool self_deleting, - P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) + typedef R (T::*MethodType)(P1, P2, P3, P4, P5, P6, A1, A2); + MethodResultCallback_6_2(T* object, MethodType method, bool self_deleting, + P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6) : object_(object), method_(method), self_deleting_(self_deleting), @@ -384,12 +386,13 @@ class MethodResultCallback_5_2 : public ResultCallback2 { p2_(p2), p3_(p3), p4_(p4), - p5_(p5) {} - ~MethodResultCallback_5_2() {} + p5_(p5), + p6_(p6) {} + ~MethodResultCallback_6_2() {} - R Run(A1 a1, A2 a2) { + R Run(A1 a1, A2 a2) override { bool needs_delete = self_deleting_; - R result = (object_->*method_)(p1_, p2_, p3_, p4_, p5_, a1, a2); + R result = (object_->*method_)(p1_, p2_, p3_, p4_, p5_, p6_, a1, a2); if (needs_delete) delete this; return result; } @@ -403,6 +406,7 @@ class MethodResultCallback_5_2 : public ResultCallback2 { typename std::remove_reference::type p3_; typename std::remove_reference::type p4_; typename std::remove_reference::type p5_; + typename std::remove_reference::type p6_; }; } // namespace internal @@ -551,27 +555,29 @@ inline ResultCallback* NewPermanentCallback( return new internal::MethodResultCallback_0_0(object, function, false); } -// See MethodResultCallback_5_2 +// See MethodResultCallback_6_2 template + typename P4, typename P5, typename P6, typename A1, typename A2> inline ResultCallback2* NewPermanentCallback( - T* object, R (T::*function)(P1, P2, P3, P4, P5, A1, A2), + T* object, R (T::*function)(P1, P2, P3, P4, P5, P6, A1, A2), typename internal::InternalConstRef::type p1, typename internal::InternalConstRef::type p2, typename internal::InternalConstRef::type p3, typename internal::InternalConstRef::type p4, - typename internal::InternalConstRef::type p5) { - return new internal::MethodResultCallback_5_2(object, function, false, p1, - p2, p3, p4, p5); + typename internal::InternalConstRef::type p5, + typename internal::InternalConstRef::type p6) { + return new internal::MethodResultCallback_6_2(object, function, false, + p1, p2, p3, p4, p5, p6); } // A function which does nothing. Useful for creating no-op callbacks, e.g.: // Closure* nothing = NewCallback(&DoNothing); -void LIBPROTOBUF_EXPORT DoNothing(); - +void PROTOBUF_EXPORT DoNothing(); } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_CALLBACK_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/casts.h b/third_party/protobuf-lite/google/protobuf/stubs/casts.h index 35e2dba0..83750bd7 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/casts.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/casts.h @@ -31,13 +31,15 @@ #ifndef GOOGLE_PROTOBUF_CASTS_H__ #define GOOGLE_PROTOBUF_CASTS_H__ -#include - #include +#include +#include + namespace google { namespace protobuf { namespace internal { + // Use implicit_cast as a safe version of static_cast or const_cast // for upcasting in the type hierarchy (i.e. casting a pointer to Foo // to a pointer to SuperclassOfFoo or casting a pointer to Foo to @@ -88,8 +90,8 @@ inline To down_cast(From* f) { // so we only accept pointers implicit_cast(0); } -#if !defined(NDEBUG) && !defined(GOOGLE_PROTOBUF_NO_RTTI) - assert(f == NULL || dynamic_cast(f) != NULL); // RTTI: debug mode only! +#if !defined(NDEBUG) && PROTOBUF_RTTI + assert(f == nullptr || dynamic_cast(f) != nullptr); // RTTI: debug mode only! #endif return static_cast(f); } @@ -105,9 +107,9 @@ inline To down_cast(From& f) { implicit_cast(0); } -#if !defined(NDEBUG) && !defined(GOOGLE_PROTOBUF_NO_RTTI) +#if !defined(NDEBUG) && PROTOBUF_RTTI // RTTI: debug mode only! - assert(dynamic_cast(&f) != NULL); + assert(dynamic_cast(&f) != nullptr); #endif return *static_cast(&f); } @@ -131,4 +133,7 @@ using internal::bit_cast; } // namespace protobuf } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_CASTS_H__ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/common.h b/third_party/protobuf-lite/google/protobuf/stubs/common.h index d35377ae..5843b5ba 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/common.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/common.h @@ -47,11 +47,6 @@ #include #include -// TODO(liujisi): Remove the following includes after the include clean-up. -#include -#include -#include - #ifndef PROTOBUF_USE_EXCEPTIONS #if defined(_MSC_VER) && defined(_CPPUNWIND) #define PROTOBUF_USE_EXCEPTIONS 1 @@ -73,22 +68,7 @@ #include #endif -#if defined(_WIN32) && defined(GetMessage) -// Allow GetMessage to be used as a valid method name in protobuf classes. -// windows.h defines GetMessage() as a macro. Let's re-define it as an inline -// function. The inline function should be equivalent for C++ users. -inline BOOL GetMessage_Win32( - LPMSG lpMsg, HWND hWnd, - UINT wMsgFilterMin, UINT wMsgFilterMax) { - return GetMessage(lpMsg, hWnd, wMsgFilterMin, wMsgFilterMax); -} -#undef GetMessage -inline BOOL GetMessage( - LPMSG lpMsg, HWND hWnd, - UINT wMsgFilterMin, UINT wMsgFilterMax) { - return GetMessage_Win32(lpMsg, hWnd, wMsgFilterMin, wMsgFilterMax); -} -#endif +#include namespace std {} @@ -101,35 +81,31 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3006001 +#define GOOGLE_PROTOBUF_VERSION 3012003 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" -// The minimum library version which works with the current version of the -// headers. -#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3006001 - // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code // generator. -static const int kMinHeaderVersionForLibrary = 3006001; +static const int kMinHeaderVersionForLibrary = 3012000; // The minimum protoc version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3006001 +#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3012000 // The minimum header version which works with the current version of // protoc. This constant should only be used in VerifyVersion(). -static const int kMinHeaderVersionForProtoc = 3006001; +static const int kMinHeaderVersionForProtoc = 3012000; // Verifies that the headers and libraries are compatible. Use the macro // below to call this. -void LIBPROTOBUF_EXPORT VerifyVersion(int headerVersion, int minLibraryVersion, - const char* filename); +void PROTOBUF_EXPORT VerifyVersion(int headerVersion, int minLibraryVersion, + const char* filename); // Converts a numeric version number to a string. -std::string LIBPROTOBUF_EXPORT VersionString(int version); +std::string PROTOBUF_EXPORT VersionString(int version); } // namespace internal @@ -151,14 +127,14 @@ namespace internal { // Checks if the buffer contains structurally-valid UTF-8. Implemented in // structurally_valid.cc. -LIBPROTOBUF_EXPORT bool IsStructurallyValidUTF8(const char* buf, int len); +PROTOBUF_EXPORT bool IsStructurallyValidUTF8(const char* buf, int len); inline bool IsStructurallyValidUTF8(const std::string& str) { return IsStructurallyValidUTF8(str.data(), static_cast(str.length())); } -// Returns initial number of bytes of structually valid UTF-8. -LIBPROTOBUF_EXPORT int UTF8SpnStructurallyValid(const StringPiece& str); +// Returns initial number of bytes of structurally valid UTF-8. +PROTOBUF_EXPORT int UTF8SpnStructurallyValid(const StringPiece& str); // Coerce UTF-8 byte string in src_str to be // a structurally-valid equal-length string by selectively @@ -172,42 +148,24 @@ LIBPROTOBUF_EXPORT int UTF8SpnStructurallyValid(const StringPiece& str); // // Optimized for: all structurally valid and no byte copying is done. // -LIBPROTOBUF_EXPORT char* UTF8CoerceToStructurallyValid( - const StringPiece& str, char* dst, char replace_char); +PROTOBUF_EXPORT char* UTF8CoerceToStructurallyValid(const StringPiece& str, + char* dst, + char replace_char); } // namespace internal - -// =================================================================== -// Shutdown support. - -// Shut down the entire protocol buffers library, deleting all static-duration -// objects allocated by the library or by generated .pb.cc files. -// -// There are two reasons you might want to call this: -// * You use a draconian definition of "memory leak" in which you expect -// every single malloc() to have a corresponding free(), even for objects -// which live until program exit. -// * You are writing a dynamically-loaded library which needs to clean up -// after itself when the library is unloaded. -// -// It is safe to call this multiple times. However, it is not safe to use -// any other part of the protocol buffers library after -// ShutdownProtobufLibrary() has been called. Furthermore this call is not -// thread safe, user needs to synchronize multiple calls. -LIBPROTOBUF_EXPORT void ShutdownProtobufLibrary(); +// This lives in message_lite.h now, but we leave this here for any users that +// #include common.h and not message_lite.h. +PROTOBUF_EXPORT void ShutdownProtobufLibrary(); namespace internal { -// Register a function to be called when ShutdownProtocolBuffers() is called. -LIBPROTOBUF_EXPORT void OnShutdown(void (*func)()); -// Run an arbitrary function on an arg -LIBPROTOBUF_EXPORT void OnShutdownRun(void (*f)(const void*), const void* arg); - +// Strongly references the given variable such that the linker will be forced +// to pull in this variable's translation unit. template -T* OnShutdownDelete(T* p) { - OnShutdownRun([](const void* p) { delete static_cast(p); }, p); - return p; +void StrongReference(const T& var) { + auto volatile unused = &var; + (void)&unused; // Use address to avoid an extra load of "unused". } } // namespace internal @@ -239,4 +197,6 @@ using std::string; } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_COMMON_H__ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/fastmem.h b/third_party/protobuf-lite/google/protobuf/stubs/fastmem.h index 1f1f6ed3..76c8a3ae 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/fastmem.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/fastmem.h @@ -51,6 +51,8 @@ #include +#include + namespace google { namespace protobuf { namespace internal { @@ -64,7 +66,7 @@ namespace internal { // if it makes sense to do so.:w inline bool memeq(const char* a, const char* b, size_t n) { size_t n_rounded_down = n & ~static_cast(7); - if (GOOGLE_PREDICT_FALSE(n_rounded_down == 0)) { // n <= 7 + if (PROTOBUF_PREDICT_FALSE(n_rounded_down == 0)) { // n <= 7 return memcmp(a, b, n) == 0; } // n >= 8 @@ -150,4 +152,6 @@ inline void memcpy_inlined(char *dst, const char *src, size_t size) { } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_FASTMEM_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/hash.h b/third_party/protobuf-lite/google/protobuf/stubs/hash.h index fd8ba156..a093b406 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/hash.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/hash.h @@ -29,8 +29,6 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Author: kenton@google.com (Kenton Varda) -// -// Deals with the fact that hash_map is not defined everywhere. #ifndef GOOGLE_PROTOBUF_STUBS_HASH_H__ #define GOOGLE_PROTOBUF_STUBS_HASH_H__ @@ -38,304 +36,19 @@ #include #include -#define GOOGLE_PROTOBUF_HAVE_HASH_MAP 1 -#define GOOGLE_PROTOBUF_HAVE_HASH_SET 1 - -// Use C++11 unordered_{map|set} if available. -#if ((defined(_LIBCPP_STD_VER) && _LIBCPP_STD_VER >= 11) || \ - (((__cplusplus >= 201103L) || defined(__GXX_EXPERIMENTAL_CXX0X)) && \ - (__GLIBCXX__ > 20090421))) -# define GOOGLE_PROTOBUF_HAS_CXX11_HASH - -// For XCode >= 4.6: the compiler is clang with libc++. -// For earlier XCode version: the compiler is gcc-4.2.1 with libstdc++. -// libc++ provides and friends even in non C++11 mode, -// and it does not provide the tr1 library. Therefore the following macro -// checks against this special case. -// Note that we should not test the __APPLE_CC__ version number or the -// __clang__ macro, since the new compiler can still use -stdlib=libstdc++, in -// which case is not compilable without -std=c++11 -#elif defined(__APPLE_CC__) -# if __GNUC__ >= 4 -# define GOOGLE_PROTOBUF_HAS_TR1 -# else -// Not tested for gcc < 4... These setting can compile under 4.2.1 though. -# define GOOGLE_PROTOBUF_HASH_NAMESPACE __gnu_cxx -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS hash_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS hash_set -# endif - -// Version checks for gcc. -#elif defined(__GNUC__) -// For GCC 4.x+, use tr1::unordered_map/set; otherwise, follow the -// instructions from: -// https://gcc.gnu.org/onlinedocs/libstdc++/manual/backwards.html -# if __GNUC__ >= 4 -# define GOOGLE_PROTOBUF_HAS_TR1 -# elif __GNUC__ >= 3 -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS hash_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS hash_set -# if __GNUC__ == 3 && __GNUC_MINOR__ == 0 -# define GOOGLE_PROTOBUF_HASH_NAMESPACE std // GCC 3.0 -# else -# define GOOGLE_PROTOBUF_HASH_NAMESPACE __gnu_cxx // GCC 3.1 and later -# endif -# else -# define GOOGLE_PROTOBUF_HASH_NAMESPACE -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS hash_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS hash_set -# endif - -// GCC <= 4.1 does not define std::tr1::hash for `long long int` or `long long unsigned int` -# if __GNUC__ == 4 && defined(__GNUC_MINOR__) && __GNUC_MINOR__ <= 1 -# undef GOOGLE_PROTOBUF_HAS_TR1 -# undef GOOGLE_PROTOBUF_HAVE_HASH_MAP -# undef GOOGLE_PROTOBUF_HAVE_HASH_SET -# endif - -// Version checks for MSC. -// Apparently Microsoft decided to move hash_map *back* to the std namespace in -// MSVC 2010: -// http://blogs.msdn.com/vcblog/archive/2009/05/25/stl-breaking-changes-in-visual-studio-2010-beta-1.aspx -// And.. they are moved back to stdext in MSVC 2013 (haven't checked 2012). That -// said, use unordered_map for MSVC 2010 and beyond is our safest bet. -#elif defined(_MSC_VER) -# if _MSC_VER >= 1600 // Since Visual Studio 2010 -# define GOOGLE_PROTOBUF_HAS_CXX11_HASH -# define GOOGLE_PROTOBUF_HASH_COMPARE std::hash_compare -# elif _MSC_VER >= 1500 // Since Visual Studio 2008 -# define GOOGLE_PROTOBUF_HASH_NAMESPACE stdext -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS hash_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS hash_set -# define GOOGLE_PROTOBUF_HASH_COMPARE stdext::hash_compare -# define GOOGLE_PROTOBUF_CONTAINERS_NEED_HASH_COMPARE -# elif _MSC_VER >= 1310 -# define GOOGLE_PROTOBUF_HASH_NAMESPACE stdext -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS hash_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS hash_set -# define GOOGLE_PROTOBUF_HASH_COMPARE stdext::hash_compare -# else -# define GOOGLE_PROTOBUF_HASH_NAMESPACE std -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS hash_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS hash_set -# define GOOGLE_PROTOBUF_HASH_COMPARE stdext::hash_compare -# endif - -// **ADD NEW COMPILERS SUPPORT HERE.** -// For other compilers, undefine the macro and fallback to use std::map, in -// google/protobuf/stubs/hash.h -#else -# undef GOOGLE_PROTOBUF_HAVE_HASH_MAP -# undef GOOGLE_PROTOBUF_HAVE_HASH_SET -#endif - -#if defined(GOOGLE_PROTOBUF_HAS_CXX11_HASH) -# define GOOGLE_PROTOBUF_HASH_NAMESPACE std -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS unordered_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS unordered_set -#elif defined(GOOGLE_PROTOBUF_HAS_TR1) -# define GOOGLE_PROTOBUF_HASH_NAMESPACE std::tr1 -# include -# define GOOGLE_PROTOBUF_HASH_MAP_CLASS unordered_map -# include -# define GOOGLE_PROTOBUF_HASH_SET_CLASS unordered_set -#endif +#include +#include # define GOOGLE_PROTOBUF_HASH_NAMESPACE_DECLARATION_START \ namespace google { \ namespace protobuf { # define GOOGLE_PROTOBUF_HASH_NAMESPACE_DECLARATION_END }} -#undef GOOGLE_PROTOBUF_HAS_CXX11_HASH -#undef GOOGLE_PROTOBUF_HAS_TR1 - -#if defined(GOOGLE_PROTOBUF_HAVE_HASH_MAP) && \ - defined(GOOGLE_PROTOBUF_HAVE_HASH_SET) -#else -#define GOOGLE_PROTOBUF_MISSING_HASH -#include -#include -#endif - namespace google { namespace protobuf { -#ifdef GOOGLE_PROTOBUF_MISSING_HASH -#undef GOOGLE_PROTOBUF_MISSING_HASH - -// This system doesn't have hash_map or hash_set. Emulate them using map and -// set. - -// Make hash be the same as less. Note that everywhere where custom -// hash functions are defined in the protobuf code, they are also defined such -// that they can be used as "less" functions, which is required by MSVC anyway. template -struct hash { - // Dummy, just to make derivative hash functions compile. - int operator()(const Key& key) { - GOOGLE_LOG(FATAL) << "Should never be called."; - return 0; - } - - inline bool operator()(const Key& a, const Key& b) const { - return a < b; - } -}; - -// Make sure char* is compared by value. -template <> -struct hash { - // Dummy, just to make derivative hash functions compile. - int operator()(const char* key) { - GOOGLE_LOG(FATAL) << "Should never be called."; - return 0; - } - - inline bool operator()(const char* a, const char* b) const { - return strcmp(a, b) < 0; - } -}; - -template , - typename EqualKey = std::equal_to, - typename Alloc = std::allocator< std::pair > > -class hash_map : public std::map { - typedef std::map BaseClass; - - public: - hash_map(int a = 0, const HashFcn& b = HashFcn(), - const EqualKey& c = EqualKey(), - const Alloc& d = Alloc()) : BaseClass(b, d) {} - - HashFcn hash_function() const { return HashFcn(); } -}; - -template , - typename EqualKey = std::equal_to > -class hash_set : public std::set { - public: - hash_set(int = 0) {} - - HashFcn hash_function() const { return HashFcn(); } -}; - -#elif defined(_MSC_VER) && !defined(_STLPORT_VERSION) && \ - !(defined(_LIBCPP_STD_VER) && _LIBCPP_STD_VER >= 11) - -template -struct hash : public GOOGLE_PROTOBUF_HASH_COMPARE { -}; - -// MSVC's hash_compare hashes based on the string contents but -// compares based on the string pointer. WTF? -class CstringLess { - public: - inline bool operator()(const char* a, const char* b) const { - return strcmp(a, b) < 0; - } -}; - -template <> -struct hash - : public GOOGLE_PROTOBUF_HASH_COMPARE {}; - -#ifdef GOOGLE_PROTOBUF_CONTAINERS_NEED_HASH_COMPARE - -template -struct InternalHashCompare : public GOOGLE_PROTOBUF_HASH_COMPARE { - InternalHashCompare() {} - InternalHashCompare(HashFcn hashfcn, EqualKey equalkey) - : hashfcn_(hashfcn), equalkey_(equalkey) {} - size_t operator()(const Key& key) const { return hashfcn_(key); } - bool operator()(const Key& key1, const Key& key2) const { - return !equalkey_(key1, key2); - } - HashFcn hashfcn_; - EqualKey equalkey_; -}; - -template , - typename EqualKey = std::equal_to, - typename Alloc = std::allocator< std::pair > > -class hash_map - : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_MAP_CLASS< - Key, Data, InternalHashCompare, Alloc> { - typedef GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_MAP_CLASS< - Key, Data, InternalHashCompare, Alloc> BaseClass; - - public: - hash_map(int a = 0, const HashFcn& b = HashFcn(), - const EqualKey& c = EqualKey(), const Alloc& d = Alloc()) - : BaseClass(InternalHashCompare(b, c), d) {} - - HashFcn hash_function() const { return HashFcn(); } -}; - -template , - typename EqualKey = std::equal_to > -class hash_set - : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_SET_CLASS< - Key, InternalHashCompare > { - public: - hash_set(int = 0) {} - - HashFcn hash_function() const { return HashFcn(); } -}; - -#else // GOOGLE_PROTOBUF_CONTAINERS_NEED_HASH_COMPARE - -template , - typename EqualKey = std::equal_to, - typename Alloc = std::allocator< std::pair > > -class hash_map - : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_MAP_CLASS< - Key, Data, HashFcn, EqualKey, Alloc> { - typedef GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_MAP_CLASS< - Key, Data, HashFcn, EqualKey, Alloc> BaseClass; - - public: - hash_map(int a = 0, const HashFcn& b = HashFcn(), - const EqualKey& c = EqualKey(), - const Alloc& d = Alloc()) : BaseClass(a, b, c, d) {} - - HashFcn hash_function() const { return HashFcn(); } -}; - -template , - typename EqualKey = std::equal_to > -class hash_set - : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_SET_CLASS< - Key, HashFcn, EqualKey> { - public: - hash_set(int = 0) {} - - HashFcn hash_function() const { return HashFcn(); } -}; -#endif // GOOGLE_PROTOBUF_CONTAINERS_NEED_HASH_COMPARE - -#else // defined(_MSC_VER) && !defined(_STLPORT_VERSION) - -template -struct hash : public GOOGLE_PROTOBUF_HASH_NAMESPACE::hash { -}; +struct hash : public std::hash {}; template struct hash { @@ -364,37 +77,6 @@ struct hash { } }; -template , - typename EqualKey = std::equal_to, - typename Alloc = std::allocator< std::pair > > -class hash_map - : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_MAP_CLASS< - Key, Data, HashFcn, EqualKey, Alloc> { - typedef GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_MAP_CLASS< - Key, Data, HashFcn, EqualKey, Alloc> BaseClass; - - public: - hash_map(int a = 0, const HashFcn& b = HashFcn(), - const EqualKey& c = EqualKey(), - const Alloc& d = Alloc()) : BaseClass(a, b, c, d) {} - - HashFcn hash_function() const { return HashFcn(); } -}; - -template , - typename EqualKey = std::equal_to > -class hash_set - : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_SET_CLASS< - Key, HashFcn, EqualKey> { - public: - hash_set(int = 0) {} - - HashFcn hash_function() const { return HashFcn(); } -}; - -#endif // !GOOGLE_PROTOBUF_MISSING_HASH - template <> struct hash { inline size_t operator()(const string& key) const { diff --git a/third_party/protobuf-lite/google/protobuf/stubs/int128.h b/third_party/protobuf-lite/google/protobuf/stubs/int128.h index 1499bb76..dc70d96e 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/int128.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/int128.h @@ -34,6 +34,8 @@ #include +#include + namespace google { namespace protobuf { @@ -48,7 +50,7 @@ struct uint128_pod; #endif // An unsigned 128-bit integer type. Thread-compatible. -class LIBPROTOBUF_EXPORT uint128 { +class PROTOBUF_EXPORT uint128 { public: UINT128_CONSTEXPR uint128(); // Sets to 0, but don't trust on this behavior. UINT128_CONSTEXPR uint128(uint64 top, uint64 bottom); @@ -84,8 +86,8 @@ class LIBPROTOBUF_EXPORT uint128 { friend uint64 Uint128High64(const uint128& v); // We add "std::" to avoid including all of port.h. - LIBPROTOBUF_EXPORT friend std::ostream& operator<<(std::ostream& o, - const uint128& b); + PROTOBUF_EXPORT friend std::ostream& operator<<(std::ostream& o, + const uint128& b); private: static void DivModImpl(uint128 dividend, uint128 divisor, @@ -116,11 +118,11 @@ struct uint128_pod { uint64 lo; }; -LIBPROTOBUF_EXPORT extern const uint128_pod kuint128max; +PROTOBUF_EXPORT extern const uint128_pod kuint128max; // allow uint128 to be logged -LIBPROTOBUF_EXPORT extern std::ostream& operator<<(std::ostream& o, - const uint128& b); +PROTOBUF_EXPORT extern std::ostream& operator<<(std::ostream& o, + const uint128& b); // Methods to access low and high pieces of 128-bit value. // Defined externally from uint128 to facilitate conversion @@ -380,4 +382,6 @@ inline uint128& uint128::operator--() { } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_INT128_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/logging.h b/third_party/protobuf-lite/google/protobuf/stubs/logging.h index f69605d9..f37048d6 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/logging.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/logging.h @@ -34,6 +34,8 @@ #include #include +#include + // =================================================================== // emulates google3/base/logging.h @@ -70,7 +72,7 @@ namespace internal { class LogFinisher; -class LIBPROTOBUF_EXPORT LogMessage { +class PROTOBUF_EXPORT LogMessage { public: LogMessage(LogLevel level, const char* filename, int line); ~LogMessage(); @@ -87,7 +89,7 @@ class LIBPROTOBUF_EXPORT LogMessage { LogMessage& operator<<(double value); LogMessage& operator<<(void* value); LogMessage& operator<<(const StringPiece& value); - LogMessage& operator<<(const ::google::protobuf::util::Status& status); + LogMessage& operator<<(const util::Status& status); LogMessage& operator<<(const uint128& value); private: @@ -102,7 +104,7 @@ class LIBPROTOBUF_EXPORT LogMessage { // Used to make the entire "LOG(BLAH) << etc." expression have a void return // type and print a newline after each message. -class LIBPROTOBUF_EXPORT LogFinisher { +class PROTOBUF_EXPORT LogFinisher { public: void operator=(LogMessage& other); }; @@ -141,10 +143,10 @@ inline bool IsOk(bool status) { return status; } #undef GOOGLE_DCHECK_GT #undef GOOGLE_DCHECK_GE -#define GOOGLE_LOG(LEVEL) \ - ::google::protobuf::internal::LogFinisher() = \ - ::google::protobuf::internal::LogMessage( \ - ::google::protobuf::LOGLEVEL_##LEVEL, __FILE__, __LINE__) +#define GOOGLE_LOG(LEVEL) \ + ::google::protobuf::internal::LogFinisher() = \ + ::google::protobuf::internal::LogMessage( \ + ::google::protobuf::LOGLEVEL_##LEVEL, __FILE__, __LINE__) #define GOOGLE_LOG_IF(LEVEL, CONDITION) \ !(CONDITION) ? (void)0 : GOOGLE_LOG(LEVEL) @@ -162,15 +164,15 @@ namespace internal { template T* CheckNotNull(const char* /* file */, int /* line */, const char* name, T* val) { - if (val == NULL) { + if (val == nullptr) { GOOGLE_LOG(FATAL) << name; } return val; } } // namespace internal -#define GOOGLE_CHECK_NOTNULL(A) \ - ::google::protobuf::internal::CheckNotNull(\ - __FILE__, __LINE__, "'" #A "' must not be NULL", (A)) +#define GOOGLE_CHECK_NOTNULL(A) \ + ::google::protobuf::internal::CheckNotNull( \ + __FILE__, __LINE__, "'" #A "' must not be nullptr", (A)) #ifdef NDEBUG @@ -208,7 +210,7 @@ typedef void LogHandler(LogLevel level, const char* filename, int line, // also help end users figure out a problem. If you would prefer that // these messages be sent somewhere other than stderr, call SetLogHandler() // to set your own handler. This returns the old handler. Set the handler -// to NULL to ignore log messages (but see also LogSilencer, below). +// to nullptr to ignore log messages (but see also LogSilencer, below). // // Obviously, SetLogHandler is not thread-safe. You should only call it // at initialization time, and probably not from library code. If you @@ -216,7 +218,7 @@ typedef void LogHandler(LogLevel level, const char* filename, int line, // have some code that tends to trigger them frequently and you know // the warnings are not important to you), use the LogSilencer class // below. -LIBPROTOBUF_EXPORT LogHandler* SetLogHandler(LogHandler* new_func); +PROTOBUF_EXPORT LogHandler* SetLogHandler(LogHandler* new_func); // Create a LogSilencer if you want to temporarily suppress all log // messages. As long as any LogSilencer objects exist, non-fatal @@ -225,7 +227,7 @@ LIBPROTOBUF_EXPORT LogHandler* SetLogHandler(LogHandler* new_func); // accidentally suppress log messages occurring in another thread, but // since messages are generally for debugging purposes only, this isn't // a big deal. If you want to intercept log messages, use SetLogHandler(). -class LIBPROTOBUF_EXPORT LogSilencer { +class PROTOBUF_EXPORT LogSilencer { public: LogSilencer(); ~LogSilencer(); @@ -234,4 +236,6 @@ class LIBPROTOBUF_EXPORT LogSilencer { } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_LOGGING_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/macros.h b/third_party/protobuf-lite/google/protobuf/stubs/macros.h index 0e9a9ec1..c556d022 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/macros.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/macros.h @@ -112,55 +112,7 @@ struct CompileAssert { } // namespace internal -#undef GOOGLE_COMPILE_ASSERT -#if __cplusplus >= 201103L #define GOOGLE_COMPILE_ASSERT(expr, msg) static_assert(expr, #msg) -#else -#define GOOGLE_COMPILE_ASSERT(expr, msg) \ - ::google::protobuf::internal::CompileAssert<(bool(expr))> \ - msg[bool(expr) ? 1 : -1]; \ - (void)msg -// Implementation details of COMPILE_ASSERT: -// -// - COMPILE_ASSERT works by defining an array type that has -1 -// elements (and thus is invalid) when the expression is false. -// -// - The simpler definition -// -// #define COMPILE_ASSERT(expr, msg) typedef char msg[(expr) ? 1 : -1] -// -// does not work, as gcc supports variable-length arrays whose sizes -// are determined at run-time (this is gcc's extension and not part -// of the C++ standard). As a result, gcc fails to reject the -// following code with the simple definition: -// -// int foo; -// COMPILE_ASSERT(foo, msg); // not supposed to compile as foo is -// // not a compile-time constant. -// -// - By using the type CompileAssert<(bool(expr))>, we ensures that -// expr is a compile-time constant. (Template arguments must be -// determined at compile-time.) -// -// - The outter parentheses in CompileAssert<(bool(expr))> are necessary -// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written -// -// CompileAssert -// -// instead, these compilers will refuse to compile -// -// COMPILE_ASSERT(5 > 0, some_message); -// -// (They seem to think the ">" in "5 > 0" marks the end of the -// template argument list.) -// -// - The array size is (bool(expr) ? 1 : -1), instead of simply -// -// ((expr) ? 1 : -1). -// -// This is to avoid running into a bug in MS VC 7.1, which -// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1. -#endif // __cplusplus >= 201103L } // namespace protobuf } // namespace google diff --git a/third_party/protobuf-lite/google/protobuf/stubs/map_util.h b/third_party/protobuf-lite/google/protobuf/stubs/map_util.h index 3e6d381f..24e098ad 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/map_util.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/map_util.h @@ -131,7 +131,7 @@ FindWithDefault(const Collection& collection, } // Returns a pointer to the const value associated with the given key if it -// exists, or NULL otherwise. +// exists, or nullptr otherwise. template const typename Collection::value_type::second_type* FindOrNull(const Collection& collection, @@ -156,11 +156,11 @@ FindOrNull(Collection& collection, // NOLINT } // Returns the pointer value associated with the given key. If none is found, -// NULL is returned. The function is designed to be used with a map of keys to +// nullptr is returned. The function is designed to be used with a map of keys to // pointers. // // This function does not distinguish between a missing key and a key mapped -// to a NULL value. +// to nullptr. template typename Collection::value_type::second_type FindPtrOrNull(const Collection& collection, @@ -188,7 +188,7 @@ FindPtrOrNull(Collection& collection, // NOLINT } // Finds the pointer value associated with the given key in a map whose values -// are linked_ptrs. Returns NULL if key is not found. +// are linked_ptrs. Returns nullptr if key is not found. template typename Collection::value_type::second_type::element_type* FindLinkedPtrOrNull(const Collection& collection, @@ -215,7 +215,7 @@ FindLinkedPtrOrDie(const Collection& collection, } // Finds the value associated with the given key and copies it to *value (if not -// NULL). Returns false if the key was not found, true otherwise. +// nullptr). Returns false if the key was not found, true otherwise. template bool FindCopy(const Collection& collection, const Key& key, @@ -447,7 +447,7 @@ LookupOrInsertNew(Collection* const collection, std::pair ret = collection->insert(typename Collection::value_type( key, - static_cast(NULL))); + static_cast(nullptr))); if (ret.second) { ret.first->second = new Element(); } @@ -466,7 +466,7 @@ LookupOrInsertNew(Collection* const collection, std::pair ret = collection->insert(typename Collection::value_type( key, - static_cast(NULL))); + static_cast(nullptr))); if (ret.second) { ret.first->second = new Element(arg); } @@ -612,7 +612,7 @@ bool UpdateReturnCopy(Collection* const collection, return false; } -// Tries to insert the given key-value pair into the collection. Returns NULL if +// Tries to insert the given key-value pair into the collection. Returns nullptr if // the insert succeeds. Otherwise, returns a pointer to the existing value. // // This complements UpdateReturnCopy in that it allows to update only after @@ -620,12 +620,11 @@ bool UpdateReturnCopy(Collection* const collection, // twice. Unlike UpdateReturnCopy this also does not come with the issue of an // undefined previous* in case new data was inserted. template -typename Collection::value_type::second_type* const -InsertOrReturnExisting(Collection* const collection, - const typename Collection::value_type& vt) { +typename Collection::value_type::second_type* InsertOrReturnExisting( + Collection* const collection, const typename Collection::value_type& vt) { std::pair ret = collection->insert(vt); if (ret.second) { - return NULL; // Inserted, no existing previous value. + return nullptr; // Inserted, no existing previous value. } else { return &ret.first->second; // Return address of already existing value. } @@ -633,8 +632,7 @@ InsertOrReturnExisting(Collection* const collection, // Same as above, except for explicit key and data. template -typename Collection::value_type::second_type* const -InsertOrReturnExisting( +typename Collection::value_type::second_type* InsertOrReturnExisting( Collection* const collection, const typename Collection::value_type::first_type& key, const typename Collection::value_type::second_type& data) { @@ -644,7 +642,7 @@ InsertOrReturnExisting( // Erases the collection item identified by the given key, and returns the value // associated with that key. It is assumed that the value (i.e., the -// mapped_type) is a pointer. Returns NULL if the key was not found in the +// mapped_type) is a pointer. Returns nullptr if the key was not found in the // collection. // // Examples: @@ -665,7 +663,7 @@ typename Collection::value_type::second_type EraseKeyReturnValuePtr( const typename Collection::value_type::first_type& key) { typename Collection::iterator it = collection->find(key); if (it == collection->end()) { - return NULL; + return nullptr; } typename Collection::value_type::second_type v = it->second; collection->erase(it); @@ -679,7 +677,7 @@ typename Collection::value_type::second_type EraseKeyReturnValuePtr( template void InsertKeysFromMap(const MapContainer& map_container, KeyContainer* key_container) { - GOOGLE_CHECK(key_container != NULL); + GOOGLE_CHECK(key_container != nullptr); for (typename MapContainer::const_iterator it = map_container.begin(); it != map_container.end(); ++it) { key_container->insert(it->first); @@ -693,7 +691,7 @@ void InsertKeysFromMap(const MapContainer& map_container, template void AppendKeysFromMap(const MapContainer& map_container, KeyContainer* key_container) { - GOOGLE_CHECK(key_container != NULL); + GOOGLE_CHECK(key_container != nullptr); for (typename MapContainer::const_iterator it = map_container.begin(); it != map_container.end(); ++it) { key_container->push_back(it->first); @@ -710,7 +708,7 @@ void AppendKeysFromMap(const MapContainer& map_container, template void AppendKeysFromMap(const MapContainer& map_container, std::vector* key_container) { - GOOGLE_CHECK(key_container != NULL); + GOOGLE_CHECK(key_container != nullptr); // We now have the opportunity to call reserve(). Calling reserve() every // time is a bad idea for some use cases: libstdc++'s implementation of // vector<>::reserve() resizes the vector's backing store to exactly the @@ -737,7 +735,7 @@ void AppendKeysFromMap(const MapContainer& map_container, template void AppendValuesFromMap(const MapContainer& map_container, ValueContainer* value_container) { - GOOGLE_CHECK(value_container != NULL); + GOOGLE_CHECK(value_container != nullptr); for (typename MapContainer::const_iterator it = map_container.begin(); it != map_container.end(); ++it) { value_container->push_back(it->second); @@ -754,7 +752,7 @@ void AppendValuesFromMap(const MapContainer& map_container, template void AppendValuesFromMap(const MapContainer& map_container, std::vector* value_container) { - GOOGLE_CHECK(value_container != NULL); + GOOGLE_CHECK(value_container != nullptr); // See AppendKeysFromMap for why this is done. if (value_container->empty()) { value_container->reserve(map_container.size()); diff --git a/third_party/protobuf-lite/google/protobuf/stubs/mathlimits.h b/third_party/protobuf-lite/google/protobuf/stubs/mathlimits.h deleted file mode 100644 index 9c9d0e9a..00000000 --- a/third_party/protobuf-lite/google/protobuf/stubs/mathlimits.h +++ /dev/null @@ -1,303 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// All Rights Reserved. -// -// Author: Maxim Lifantsev -// -// Useful integer and floating point limits and type traits. -// -// This partially replaces/duplictes numeric_limits<> from . -// We get a Google-style class that we have a greater control over -// and thus can add new features to it or fix whatever happens to be broken in -// numeric_limits for the compilers we use. -// - -#ifndef UTIL_MATH_MATHLIMITS_H__ -#define UTIL_MATH_MATHLIMITS_H__ - -// Note that for Windows we do something different because it does not support -// the plain isinf and isnan. -#if __cplusplus >= 201103L -// GCC 4.9 has a bug that makes isinf and isnan ambigious when both -// and get pulled into the same translation unit. We use the ones in -// std:: namespace explicitly for C++11 -#include -#define GOOGLE_PROTOBUF_USE_STD_CMATH -#elif _GLIBCXX_USE_C99_MATH && !_GLIBCXX_USE_C99_FP_MACROS_DYNAMIC -// libstdc++ header undefines the global macros and put functions in -// std:: namespace even before C++11. Use the ones in std:: instead too. -#include -#define GOOGLE_PROTOBUF_USE_STD_CMATH -#else -#include -#endif - -#include - -#include - -#include - -// ========================================================================= // - -// Useful integer and floating point limits and type traits. -// This is just for the documentation; -// real members are defined in our specializations below. -namespace google { -namespace protobuf { -template struct MathLimits { - // Type name. - typedef T Type; - // Unsigned version of the Type with the same byte size. - // Same as Type for floating point and unsigned types. - typedef T UnsignedType; - // If the type supports negative values. - static const bool kIsSigned; - // If the type supports only integer values. - static const bool kIsInteger; - // Magnitude-wise smallest representable positive value. - static const Type kPosMin; - // Magnitude-wise largest representable positive value. - static const Type kPosMax; - // Smallest representable value. - static const Type kMin; - // Largest representable value. - static const Type kMax; - // Magnitude-wise smallest representable negative value. - // Present only if kIsSigned. - static const Type kNegMin; - // Magnitude-wise largest representable negative value. - // Present only if kIsSigned. - static const Type kNegMax; - // Smallest integer x such that 10^x is representable. - static const int kMin10Exp; - // Largest integer x such that 10^x is representable. - static const int kMax10Exp; - // Smallest positive value such that Type(1) + kEpsilon != Type(1) - static const Type kEpsilon; - // Typical rounding error that is enough to cover - // a few simple floating-point operations. - // Slightly larger than kEpsilon to account for a few rounding errors. - // Is zero if kIsInteger. - static const Type kStdError; - // Number of decimal digits of mantissa precision. - // Present only if !kIsInteger. - static const int kPrecisionDigits; - // Not a number, i.e. result of 0/0. - // Present only if !kIsInteger. - static const Type kNaN; - // Positive infinity, i.e. result of 1/0. - // Present only if !kIsInteger. - static const Type kPosInf; - // Negative infinity, i.e. result of -1/0. - // Present only if !kIsInteger. - static const Type kNegInf; - - // NOTE: Special floating point values behave - // in a special (but mathematically-logical) way - // in terms of (in)equalty comparison and mathematical operations - // -- see out unittest for examples. - - // Special floating point value testers. - // Present in integer types for convenience. - static bool IsFinite(const Type x); - static bool IsNaN(const Type x); - static bool IsInf(const Type x); - static bool IsPosInf(const Type x); - static bool IsNegInf(const Type x); -}; - -// ========================================================================= // - -// All #define-s below are simply to refactor the declarations of -// MathLimits template specializations. -// They are all #undef-ined below. - -// The hoop-jumping in *_INT_(MAX|MIN) below is so that the compiler does not -// get an overflow while computing the constants. - -#define SIGNED_INT_MAX(Type) \ - (((Type(1) << (sizeof(Type)*8 - 2)) - 1) + (Type(1) << (sizeof(Type)*8 - 2))) - -#define SIGNED_INT_MIN(Type) \ - (-(Type(1) << (sizeof(Type)*8 - 2)) - (Type(1) << (sizeof(Type)*8 - 2))) - -#define UNSIGNED_INT_MAX(Type) \ - (((Type(1) << (sizeof(Type)*8 - 1)) - 1) + (Type(1) << (sizeof(Type)*8 - 1))) - -// Compile-time selected log10-related constants for integer types. -#define SIGNED_MAX_10_EXP(Type) \ - (sizeof(Type) == 1 ? 2 : ( \ - sizeof(Type) == 2 ? 4 : ( \ - sizeof(Type) == 4 ? 9 : ( \ - sizeof(Type) == 8 ? 18 : -1)))) - -#define UNSIGNED_MAX_10_EXP(Type) \ - (sizeof(Type) == 1 ? 2 : ( \ - sizeof(Type) == 2 ? 4 : ( \ - sizeof(Type) == 4 ? 9 : ( \ - sizeof(Type) == 8 ? 19 : -1)))) - -#define DECL_INT_LIMIT_FUNCS \ - static bool IsFinite(const Type /*x*/) { return true; } \ - static bool IsNaN(const Type /*x*/) { return false; } \ - static bool IsInf(const Type /*x*/) { return false; } \ - static bool IsPosInf(const Type /*x*/) { return false; } \ - static bool IsNegInf(const Type /*x*/) { return false; } - -#define DECL_SIGNED_INT_LIMITS(IntType, UnsignedIntType) \ -template<> \ -struct LIBPROTOBUF_EXPORT MathLimits { \ - typedef IntType Type; \ - typedef UnsignedIntType UnsignedType; \ - static const bool kIsSigned = true; \ - static const bool kIsInteger = true; \ - static const Type kPosMin = 1; \ - static const Type kPosMax = SIGNED_INT_MAX(Type); \ - static const Type kMin = SIGNED_INT_MIN(Type); \ - static const Type kMax = kPosMax; \ - static const Type kNegMin = -1; \ - static const Type kNegMax = kMin; \ - static const int kMin10Exp = 0; \ - static const int kMax10Exp = SIGNED_MAX_10_EXP(Type); \ - static const Type kEpsilon = 1; \ - static const Type kStdError = 0; \ - DECL_INT_LIMIT_FUNCS \ -}; - -#define DECL_UNSIGNED_INT_LIMITS(IntType) \ -template<> \ -struct LIBPROTOBUF_EXPORT MathLimits { \ - typedef IntType Type; \ - typedef IntType UnsignedType; \ - static const bool kIsSigned = false; \ - static const bool kIsInteger = true; \ - static const Type kPosMin = 1; \ - static const Type kPosMax = UNSIGNED_INT_MAX(Type); \ - static const Type kMin = 0; \ - static const Type kMax = kPosMax; \ - static const int kMin10Exp = 0; \ - static const int kMax10Exp = UNSIGNED_MAX_10_EXP(Type); \ - static const Type kEpsilon = 1; \ - static const Type kStdError = 0; \ - DECL_INT_LIMIT_FUNCS \ -}; - -DECL_SIGNED_INT_LIMITS(signed char, unsigned char) -DECL_SIGNED_INT_LIMITS(signed short int, unsigned short int) -DECL_SIGNED_INT_LIMITS(signed int, unsigned int) -DECL_SIGNED_INT_LIMITS(signed long int, unsigned long int) -DECL_SIGNED_INT_LIMITS(signed long long int, unsigned long long int) -DECL_UNSIGNED_INT_LIMITS(unsigned char) -DECL_UNSIGNED_INT_LIMITS(unsigned short int) -DECL_UNSIGNED_INT_LIMITS(unsigned int) -DECL_UNSIGNED_INT_LIMITS(unsigned long int) -DECL_UNSIGNED_INT_LIMITS(unsigned long long int) - -#undef DECL_SIGNED_INT_LIMITS -#undef DECL_UNSIGNED_INT_LIMITS -#undef SIGNED_INT_MAX -#undef SIGNED_INT_MIN -#undef UNSIGNED_INT_MAX -#undef SIGNED_MAX_10_EXP -#undef UNSIGNED_MAX_10_EXP -#undef DECL_INT_LIMIT_FUNCS - -// For non-Windows builds we use the std:: versions of isinf and isnan if they -// are available; see the comment about at the top of this file for the -// details on why we need to do this. -#ifdef GOOGLE_PROTOBUF_USE_STD_CMATH -#define ISINF std::isinf -#define ISNAN std::isnan -#else -#define ISINF isinf -#define ISNAN isnan -#endif - -// ========================================================================= // -#if WIN32 && !__MINGW32__ // Lacks built-in isnan() and isinf() -#define DECL_FP_LIMIT_FUNCS \ - static bool IsFinite(const Type x) { return _finite(x); } \ - static bool IsNaN(const Type x) { return _isnan(x); } \ - static bool IsInf(const Type x) { return (_fpclass(x) & (_FPCLASS_NINF | _FPCLASS_PINF)) != 0; } \ - static bool IsPosInf(const Type x) { return _fpclass(x) == _FPCLASS_PINF; } \ - static bool IsNegInf(const Type x) { return _fpclass(x) == _FPCLASS_NINF; } -#else -#define DECL_FP_LIMIT_FUNCS \ - static bool IsFinite(const Type x) { return !ISINF(x) && !ISNAN(x); } \ - static bool IsNaN(const Type x) { return ISNAN(x); } \ - static bool IsInf(const Type x) { return ISINF(x); } \ - static bool IsPosInf(const Type x) { return ISINF(x) && x > 0; } \ - static bool IsNegInf(const Type x) { return ISINF(x) && x < 0; } -#endif - -// We can't put floating-point constant values in the header here because -// such constants are not considered to be primitive-type constants by gcc. -// CAVEAT: Hence, they are going to be initialized only during -// the global objects construction time. -#define DECL_FP_LIMITS(FP_Type, PREFIX) \ -template<> \ -struct LIBPROTOBUF_EXPORT MathLimits { \ - typedef FP_Type Type; \ - typedef FP_Type UnsignedType; \ - static const bool kIsSigned = true; \ - static const bool kIsInteger = false; \ - static const Type kPosMin; \ - static const Type kPosMax; \ - static const Type kMin; \ - static const Type kMax; \ - static const Type kNegMin; \ - static const Type kNegMax; \ - static const int kMin10Exp = PREFIX##_MIN_10_EXP; \ - static const int kMax10Exp = PREFIX##_MAX_10_EXP; \ - static const Type kEpsilon; \ - static const Type kStdError; \ - static const int kPrecisionDigits = PREFIX##_DIG; \ - static const Type kNaN; \ - static const Type kPosInf; \ - static const Type kNegInf; \ - DECL_FP_LIMIT_FUNCS \ -}; - -DECL_FP_LIMITS(float, FLT) -DECL_FP_LIMITS(double, DBL) -DECL_FP_LIMITS(long double, LDBL) - -#undef ISINF -#undef ISNAN -#undef DECL_FP_LIMITS -#undef DECL_FP_LIMIT_FUNCS - -// ========================================================================= // -} // namespace protobuf -} // namespace google - -#endif // UTIL_MATH_MATHLIMITS_H__ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/mutex.h b/third_party/protobuf-lite/google/protobuf/stubs/mutex.h index b9b7d2e1..b222ff74 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/mutex.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/mutex.h @@ -32,8 +32,35 @@ #include +#ifdef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + +#include + +// GetMessage conflicts with GeneratedMessageReflection::GetMessage(). +#ifdef GetMessage +#undef GetMessage +#endif + +#endif + #include +// Define thread-safety annotations for use below, if we are building with +// Clang. +#if defined(__clang__) && !defined(SWIG) +#define GOOGLE_PROTOBUF_ACQUIRE(...) \ + __attribute__((acquire_capability(__VA_ARGS__))) +#define GOOGLE_PROTOBUF_RELEASE(...) \ + __attribute__((release_capability(__VA_ARGS__))) +#define GOOGLE_PROTOBUF_CAPABILITY(x) __attribute__((capability(x))) +#else +#define GOOGLE_PROTOBUF_ACQUIRE(...) +#define GOOGLE_PROTOBUF_RELEASE(...) +#define GOOGLE_PROTOBUF_CAPABILITY(x) +#endif + +#include + // =================================================================== // emulates google3/base/mutex.h namespace google { @@ -42,26 +69,51 @@ namespace internal { #define GOOGLE_PROTOBUF_LINKER_INITIALIZED +#ifdef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + +// This class is a lightweight replacement for std::mutex on Windows platforms. +// std::mutex does not work on Windows XP SP2 with the latest VC++ libraries, +// because it utilizes the Concurrency Runtime that is only supported on Windows +// XP SP3 and above. +class PROTOBUF_EXPORT CriticalSectionLock { + public: + CriticalSectionLock() { InitializeCriticalSection(&critical_section_); } + ~CriticalSectionLock() { DeleteCriticalSection(&critical_section_); } + void lock() { EnterCriticalSection(&critical_section_); } + void unlock() { LeaveCriticalSection(&critical_section_); } + + private: + CRITICAL_SECTION critical_section_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CriticalSectionLock); +}; + +#endif + // Mutex is a natural type to wrap. As both google and other organization have // specialized mutexes. gRPC also provides an injection mechanism for custom // mutexes. -class LIBPROTOBUF_EXPORT WrappedMutex { +class GOOGLE_PROTOBUF_CAPABILITY("mutex") PROTOBUF_EXPORT WrappedMutex { public: WrappedMutex() = default; - void Lock() { mu_.lock(); } - void Unlock() { mu_.unlock(); } + void Lock() GOOGLE_PROTOBUF_ACQUIRE() { mu_.lock(); } + void Unlock() GOOGLE_PROTOBUF_RELEASE() { mu_.unlock(); } // Crash if this Mutex is not held exclusively by this thread. // May fail to crash when it should; will never crash when it should not. void AssertHeld() const {} private: +#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP std::mutex mu_; +#else // ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + CriticalSectionLock mu_; +#endif // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP }; using Mutex = WrappedMutex; // MutexLock(mu) acquires mu when constructed and releases it when destroyed. -class LIBPROTOBUF_EXPORT MutexLock { +class PROTOBUF_EXPORT MutexLock { public: explicit MutexLock(Mutex *mu) : mu_(mu) { this->mu_->Lock(); } ~MutexLock() { this->mu_->Unlock(); } @@ -74,12 +126,12 @@ class LIBPROTOBUF_EXPORT MutexLock { typedef MutexLock ReaderMutexLock; typedef MutexLock WriterMutexLock; -// MutexLockMaybe is like MutexLock, but is a no-op when mu is NULL. -class LIBPROTOBUF_EXPORT MutexLockMaybe { +// MutexLockMaybe is like MutexLock, but is a no-op when mu is nullptr. +class PROTOBUF_EXPORT MutexLockMaybe { public: explicit MutexLockMaybe(Mutex *mu) : - mu_(mu) { if (this->mu_ != NULL) { this->mu_->Lock(); } } - ~MutexLockMaybe() { if (this->mu_ != NULL) { this->mu_->Unlock(); } } + mu_(mu) { if (this->mu_ != nullptr) { this->mu_->Lock(); } } + ~MutexLockMaybe() { if (this->mu_ != nullptr) { this->mu_->Unlock(); } } private: Mutex *const mu_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MutexLockMaybe); @@ -97,7 +149,7 @@ class ThreadLocalStorage { } T* Get() { T* result = static_cast(pthread_getspecific(key_)); - if (result == NULL) { + if (result == nullptr) { result = new T(); pthread_setspecific(key_, result); } @@ -123,8 +175,12 @@ using internal::ReaderMutexLock; using internal::WriterMutexLock; using internal::MutexLockMaybe; - } // namespace protobuf } // namespace google +#undef GOOGLE_PROTOBUF_ACQUIRE +#undef GOOGLE_PROTOBUF_RELEASE + +#include + #endif // GOOGLE_PROTOBUF_STUBS_MUTEX_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h b/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h index c3a64dd2..ce1b1e36 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h @@ -56,7 +56,7 @@ #elif defined(__aarch64__) #define GOOGLE_PROTOBUF_ARCH_AARCH64 1 #define GOOGLE_PROTOBUF_ARCH_64_BIT 1 -#elif defined(__MIPSEL__) +#elif defined(__mips__) #if defined(__LP64__) #define GOOGLE_PROTOBUF_ARCH_MIPS64 1 #define GOOGLE_PROTOBUF_ARCH_64_BIT 1 @@ -99,6 +99,7 @@ GOOGLE_PROTOBUF_PLATFORM_ERROR #if defined(__APPLE__) #define GOOGLE_PROTOBUF_OS_APPLE +#include #include #if TARGET_OS_IPHONE #define GOOGLE_PROTOBUF_OS_IPHONE @@ -125,4 +126,9 @@ GOOGLE_PROTOBUF_PLATFORM_ERROR #define GOOGLE_PROTOBUF_NO_THREADLOCAL #endif +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 1070 +// __thread keyword requires at least 10.7 +#define GOOGLE_PROTOBUF_NO_THREADLOCAL +#endif + #endif // GOOGLE_PROTOBUF_PLATFORM_MACROS_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/port.h b/third_party/protobuf-lite/google/protobuf/stubs/port.h deleted file mode 100644 index 6b52305f..00000000 --- a/third_party/protobuf-lite/google/protobuf/stubs/port.h +++ /dev/null @@ -1,542 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef GOOGLE_PROTOBUF_STUBS_PORT_H_ -#define GOOGLE_PROTOBUF_STUBS_PORT_H_ - -#include -#include -#include -#include -#include -#if defined(__osf__) -// Tru64 lacks stdint.h, but has inttypes.h which defines a superset of -// what stdint.h would define. -#include -#elif !defined(_MSC_VER) -#include -#endif - -#include - -#undef PROTOBUF_LITTLE_ENDIAN -#ifdef _WIN32 - // Assuming windows is always little-endian. - // TODO(xiaofeng): The PROTOBUF_LITTLE_ENDIAN is not only used for - // optimization but also for correctness. We should define an - // different macro to test the big-endian code path in coded_stream. - #if !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) - #define PROTOBUF_LITTLE_ENDIAN 1 - #endif - #if _MSC_VER >= 1300 && !defined(__INTEL_COMPILER) - // If MSVC has "/RTCc" set, it will complain about truncating casts at - // runtime. This file contains some intentional truncating casts. - #pragma runtime_checks("c", off) - #endif -#else - #include // __BYTE_ORDER - #if defined(__OpenBSD__) - #include - #endif - #if ((defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)) || \ - (defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN) || \ - (defined(BYTE_ORDER) && BYTE_ORDER == LITTLE_ENDIAN)) && \ - !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) - #define PROTOBUF_LITTLE_ENDIAN 1 - #endif -#endif -#if defined(_MSC_VER) && defined(PROTOBUF_USE_DLLS) - #ifdef LIBPROTOBUF_EXPORTS - #define LIBPROTOBUF_EXPORT __declspec(dllexport) - #else - #define LIBPROTOBUF_EXPORT __declspec(dllimport) - #endif - #ifdef LIBPROTOC_EXPORTS - #define LIBPROTOC_EXPORT __declspec(dllexport) - #else - #define LIBPROTOC_EXPORT __declspec(dllimport) - #endif -#else - #define LIBPROTOBUF_EXPORT - #define LIBPROTOC_EXPORT -#endif - -// These #includes are for the byte swap functions declared later on. -#ifdef _MSC_VER -#include // NOLINT(build/include) -#include -#elif defined(__APPLE__) -#include -#elif defined(__GLIBC__) || defined(__BIONIC__) || defined(__CYGWIN__) -#include // IWYU pragma: export -#endif - -#define PROTOBUF_RUNTIME_DEPRECATED(message) - -// =================================================================== -// from google3/base/port.h - -#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L || \ - (defined(_MSC_VER) && _MSC_VER >= 1900)) -// Define this to 1 if the code is compiled in C++11 mode; leave it -// undefined otherwise. Do NOT define it to 0 -- that causes -// '#ifdef LANG_CXX11' to behave differently from '#if LANG_CXX11'. -#define LANG_CXX11 1 -#endif - -#if LANG_CXX11 && !defined(__NVCC__) -#define PROTOBUF_CXX11 1 -#else -#define PROTOBUF_CXX11 0 -#endif - -#if PROTOBUF_CXX11 -#define PROTOBUF_FINAL final -#else -#define PROTOBUF_FINAL -#endif - -namespace google { -namespace protobuf { - -typedef unsigned int uint; - -#ifdef _MSC_VER -typedef signed __int8 int8; -typedef __int16 int16; -typedef __int32 int32; -typedef __int64 int64; - -typedef unsigned __int8 uint8; -typedef unsigned __int16 uint16; -typedef unsigned __int32 uint32; -typedef unsigned __int64 uint64; -#else -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; -#endif - -// long long macros to be used because gcc and vc++ use different suffixes, -// and different size specifiers in format strings -#undef GOOGLE_LONGLONG -#undef GOOGLE_ULONGLONG -#undef GOOGLE_LL_FORMAT - -#ifdef _MSC_VER -#define GOOGLE_LONGLONG(x) x##I64 -#define GOOGLE_ULONGLONG(x) x##UI64 -#define GOOGLE_LL_FORMAT "I64" // As in printf("%I64d", ...) -#else -// By long long, we actually mean int64. -#define GOOGLE_LONGLONG(x) x##LL -#define GOOGLE_ULONGLONG(x) x##ULL -// Used to format real long long integers. -#define GOOGLE_LL_FORMAT "ll" // As in "%lld". Note that "q" is poor form also. -#endif - -static const int32 kint32max = 0x7FFFFFFF; -static const int32 kint32min = -kint32max - 1; -static const int64 kint64max = GOOGLE_LONGLONG(0x7FFFFFFFFFFFFFFF); -static const int64 kint64min = -kint64max - 1; -static const uint32 kuint32max = 0xFFFFFFFFu; -static const uint64 kuint64max = GOOGLE_ULONGLONG(0xFFFFFFFFFFFFFFFF); - -// ------------------------------------------------------------------- -// Annotations: Some parts of the code have been annotated in ways that might -// be useful to some compilers or tools, but are not supported universally. -// You can #define these annotations yourself if the default implementation -// is not right for you. - -#ifndef GOOGLE_ATTRIBUTE_ALWAYS_INLINE -#if defined(__GNUC__) && (__GNUC__ > 3 ||(__GNUC__ == 3 && __GNUC_MINOR__ >= 1)) -// For functions we want to force inline. -// Introduced in gcc 3.1. -#define GOOGLE_ATTRIBUTE_ALWAYS_INLINE __attribute__ ((always_inline)) -#else -// Other compilers will have to figure it out for themselves. -#define GOOGLE_ATTRIBUTE_ALWAYS_INLINE -#endif -#endif - -#define GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE GOOGLE_ATTRIBUTE_ALWAYS_INLINE - -#ifndef GOOGLE_ATTRIBUTE_NOINLINE -#if defined(__GNUC__) && (__GNUC__ > 3 ||(__GNUC__ == 3 && __GNUC_MINOR__ >= 1)) -// For functions we want to force not inline. -// Introduced in gcc 3.1. -#define GOOGLE_ATTRIBUTE_NOINLINE __attribute__ ((noinline)) -#elif defined(_MSC_VER) && (_MSC_VER >= 1400) -// Seems to have been around since at least Visual Studio 2005 -#define GOOGLE_ATTRIBUTE_NOINLINE __declspec(noinline) -#else -// Other compilers will have to figure it out for themselves. -#define GOOGLE_ATTRIBUTE_NOINLINE -#endif -#endif - -#define GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE GOOGLE_ATTRIBUTE_NOINLINE - -#ifndef GOOGLE_ATTRIBUTE_FUNC_ALIGN -#if defined(__clang__) || \ - defined(__GNUC__) && (__GNUC__ > 4 ||(__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) -// Function alignment attribute introduced in gcc 4.3 -#define GOOGLE_ATTRIBUTE_FUNC_ALIGN(bytes) __attribute__ ((aligned(bytes))) -#else -#define GOOGLE_ATTRIBUTE_FUNC_ALIGN(bytes) -#endif -#endif - -#define GOOGLE_PROTOBUF_ATTRIBUTE_FUNC_ALIGN(bytes) \ - GOOGLE_ATTRIBUTE_FUNC_ALIGN(bytes) - -#ifndef GOOGLE_PREDICT_TRUE -#ifdef __GNUC__ -// Provided at least since GCC 3.0. -#define GOOGLE_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) -#else -#define GOOGLE_PREDICT_TRUE(x) (x) -#endif -#endif - -#ifndef GOOGLE_PREDICT_FALSE -#ifdef __GNUC__ -// Provided at least since GCC 3.0. -#define GOOGLE_PREDICT_FALSE(x) (__builtin_expect(x, 0)) -#else -#define GOOGLE_PREDICT_FALSE(x) (x) -#endif -#endif - -#ifndef GOOGLE_PROTOBUF_ATTRIBUTE_RETURNS_NONNULL -#ifdef __GNUC__ -#define GOOGLE_PROTOBUF_ATTRIBUTE_RETURNS_NONNULL \ - __attribute__((returns_nonnull)) -#endif -#endif - -// Delimits a block of code which may write to memory which is simultaneously -// written by other threads, but which has been determined to be thread-safe -// (e.g. because it is an idempotent write). -#ifndef GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN -#define GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN() -#endif -#ifndef GOOGLE_SAFE_CONCURRENT_WRITES_END -#define GOOGLE_SAFE_CONCURRENT_WRITES_END() -#endif - -#define GOOGLE_GUARDED_BY(x) -#define GOOGLE_ATTRIBUTE_COLD - -#ifdef GOOGLE_PROTOBUF_DONT_USE_UNALIGNED -# define GOOGLE_PROTOBUF_USE_UNALIGNED 0 -#else -# if defined(_M_X64) || defined(__x86_64__) || defined(_M_IX86) || defined(__i386__) -# define GOOGLE_PROTOBUF_USE_UNALIGNED 1 -# else -# define GOOGLE_PROTOBUF_USE_UNALIGNED 0 -# endif -#endif - -#define GOOGLE_PROTOBUF_ATTRIBUTE_COLD GOOGLE_ATTRIBUTE_COLD - -#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\ - defined(MEMORY_SANITIZER) - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus -uint16_t __sanitizer_unaligned_load16(const void *p); -uint32_t __sanitizer_unaligned_load32(const void *p); -uint64_t __sanitizer_unaligned_load64(const void *p); -void __sanitizer_unaligned_store16(void *p, uint16_t v); -void __sanitizer_unaligned_store32(void *p, uint32_t v); -void __sanitizer_unaligned_store64(void *p, uint64_t v); -#ifdef __cplusplus -} // extern "C" -#endif // __cplusplus - -inline uint16 GOOGLE_UNALIGNED_LOAD16(const void *p) { - return __sanitizer_unaligned_load16(p); -} - -inline uint32 GOOGLE_UNALIGNED_LOAD32(const void *p) { - return __sanitizer_unaligned_load32(p); -} - -inline uint64 GOOGLE_UNALIGNED_LOAD64(const void *p) { - return __sanitizer_unaligned_load64(p); -} - -inline void GOOGLE_UNALIGNED_STORE16(void *p, uint16 v) { - __sanitizer_unaligned_store16(p, v); -} - -inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32 v) { - __sanitizer_unaligned_store32(p, v); -} - -inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64 v) { - __sanitizer_unaligned_store64(p, v); -} - -#elif GOOGLE_PROTOBUF_USE_UNALIGNED - -#define GOOGLE_UNALIGNED_LOAD16(_p) (*reinterpret_cast(_p)) -#define GOOGLE_UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) -#define GOOGLE_UNALIGNED_LOAD64(_p) (*reinterpret_cast(_p)) - -#define GOOGLE_UNALIGNED_STORE16(_p, _val) (*reinterpret_cast(_p) = (_val)) -#define GOOGLE_UNALIGNED_STORE32(_p, _val) (*reinterpret_cast(_p) = (_val)) -#define GOOGLE_UNALIGNED_STORE64(_p, _val) (*reinterpret_cast(_p) = (_val)) - -#else -inline uint16 GOOGLE_UNALIGNED_LOAD16(const void *p) { - uint16 t; - memcpy(&t, p, sizeof t); - return t; -} - -inline uint32 GOOGLE_UNALIGNED_LOAD32(const void *p) { - uint32 t; - memcpy(&t, p, sizeof t); - return t; -} - -inline uint64 GOOGLE_UNALIGNED_LOAD64(const void *p) { - uint64 t; - memcpy(&t, p, sizeof t); - return t; -} - -inline void GOOGLE_UNALIGNED_STORE16(void *p, uint16 v) { - memcpy(p, &v, sizeof v); -} - -inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32 v) { - memcpy(p, &v, sizeof v); -} - -inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64 v) { - memcpy(p, &v, sizeof v); -} -#endif - -#if defined(GOOGLE_PROTOBUF_OS_NACL) \ - || (defined(__ANDROID__) && defined(__clang__) \ - && (__clang_major__ == 3 && __clang_minor__ == 8) \ - && (__clang_patchlevel__ < 275480)) -# define GOOGLE_PROTOBUF_USE_PORTABLE_LOG2 -#endif - -#if defined(_MSC_VER) -#define GOOGLE_THREAD_LOCAL __declspec(thread) -#else -#define GOOGLE_THREAD_LOCAL __thread -#endif - -// The following guarantees declaration of the byte swap functions. -#ifdef _MSC_VER -#define bswap_16(x) _byteswap_ushort(x) -#define bswap_32(x) _byteswap_ulong(x) -#define bswap_64(x) _byteswap_uint64(x) - -#elif defined(__APPLE__) -// Mac OS X / Darwin features -#define bswap_16(x) OSSwapInt16(x) -#define bswap_32(x) OSSwapInt32(x) -#define bswap_64(x) OSSwapInt64(x) - -#elif !defined(__GLIBC__) && !defined(__BIONIC__) && !defined(__CYGWIN__) - -static inline uint16 bswap_16(uint16 x) { - return static_cast(((x & 0xFF) << 8) | ((x & 0xFF00) >> 8)); -} -#define bswap_16(x) bswap_16(x) -static inline uint32 bswap_32(uint32 x) { - return (((x & 0xFF) << 24) | - ((x & 0xFF00) << 8) | - ((x & 0xFF0000) >> 8) | - ((x & 0xFF000000) >> 24)); -} -#define bswap_32(x) bswap_32(x) -static inline uint64 bswap_64(uint64 x) { - return (((x & GOOGLE_ULONGLONG(0xFF)) << 56) | - ((x & GOOGLE_ULONGLONG(0xFF00)) << 40) | - ((x & GOOGLE_ULONGLONG(0xFF0000)) << 24) | - ((x & GOOGLE_ULONGLONG(0xFF000000)) << 8) | - ((x & GOOGLE_ULONGLONG(0xFF00000000)) >> 8) | - ((x & GOOGLE_ULONGLONG(0xFF0000000000)) >> 24) | - ((x & GOOGLE_ULONGLONG(0xFF000000000000)) >> 40) | - ((x & GOOGLE_ULONGLONG(0xFF00000000000000)) >> 56)); -} -#define bswap_64(x) bswap_64(x) - -#endif - -// =================================================================== -// from google3/util/bits/bits.h - -class Bits { - public: - static uint32 Log2FloorNonZero(uint32 n) { -#if defined(__GNUC__) - return 31 ^ static_cast(__builtin_clz(n)); -#elif defined(_MSC_VER) - unsigned long where; - _BitScanReverse(&where, n); - return where; -#else - return Log2FloorNonZero_Portable(n); -#endif - } - - static uint32 Log2FloorNonZero64(uint64 n) { - // Older versions of clang run into an instruction-selection failure when - // it encounters __builtin_clzll: - // https://bugs.chromium.org/p/nativeclient/issues/detail?id=4395 - // This includes arm-nacl-clang and clang in older Android NDK versions. - // To work around this, when we build with those we use the portable - // implementation instead. -#if defined(__GNUC__) && !defined(GOOGLE_PROTOBUF_USE_PORTABLE_LOG2) - return 63 ^ static_cast(__builtin_clzll(n)); -#elif defined(_MSC_VER) && defined(_M_X64) - unsigned long where; - _BitScanReverse64(&where, n); - return where; -#else - return Log2FloorNonZero64_Portable(n); -#endif - } - private: - static int Log2FloorNonZero_Portable(uint32 n) { - if (n == 0) - return -1; - int log = 0; - uint32 value = n; - for (int i = 4; i >= 0; --i) { - int shift = (1 << i); - uint32 x = value >> shift; - if (x != 0) { - value = x; - log += shift; - } - } - assert(value == 1); - return log; - } - - static int Log2FloorNonZero64_Portable(uint64 n) { - const uint32 topbits = static_cast(n >> 32); - if (topbits == 0) { - // Top bits are zero, so scan in bottom bits - return static_cast(Log2FloorNonZero(static_cast(n))); - } else { - return 32 + static_cast(Log2FloorNonZero(topbits)); - } - } -}; - -// =================================================================== -// from google3/util/endian/endian.h -LIBPROTOBUF_EXPORT uint32 ghtonl(uint32 x); - -class BigEndian { - public: -#ifdef PROTOBUF_LITTLE_ENDIAN - - static uint16 FromHost16(uint16 x) { return bswap_16(x); } - static uint16 ToHost16(uint16 x) { return bswap_16(x); } - - static uint32 FromHost32(uint32 x) { return bswap_32(x); } - static uint32 ToHost32(uint32 x) { return bswap_32(x); } - - static uint64 FromHost64(uint64 x) { return bswap_64(x); } - static uint64 ToHost64(uint64 x) { return bswap_64(x); } - - static bool IsLittleEndian() { return true; } - -#else - - static uint16 FromHost16(uint16 x) { return x; } - static uint16 ToHost16(uint16 x) { return x; } - - static uint32 FromHost32(uint32 x) { return x; } - static uint32 ToHost32(uint32 x) { return x; } - - static uint64 FromHost64(uint64 x) { return x; } - static uint64 ToHost64(uint64 x) { return x; } - - static bool IsLittleEndian() { return false; } - -#endif /* ENDIAN */ - - // Functions to do unaligned loads and stores in big-endian order. - static uint16 Load16(const void *p) { - return ToHost16(GOOGLE_UNALIGNED_LOAD16(p)); - } - - static void Store16(void *p, uint16 v) { - GOOGLE_UNALIGNED_STORE16(p, FromHost16(v)); - } - - static uint32 Load32(const void *p) { - return ToHost32(GOOGLE_UNALIGNED_LOAD32(p)); - } - - static void Store32(void *p, uint32 v) { - GOOGLE_UNALIGNED_STORE32(p, FromHost32(v)); - } - - static uint64 Load64(const void *p) { - return ToHost64(GOOGLE_UNALIGNED_LOAD64(p)); - } - - static void Store64(void *p, uint64 v) { - GOOGLE_UNALIGNED_STORE64(p, FromHost64(v)); - } -}; - -#ifndef GOOGLE_ATTRIBUTE_SECTION_VARIABLE -#define GOOGLE_ATTRIBUTE_SECTION_VARIABLE(name) -#endif - -#define GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(name) - -} // namespace protobuf -} // namespace google - -#endif // GOOGLE_PROTOBUF_STUBS_PORT_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/status.h b/third_party/protobuf-lite/google/protobuf/stubs/status.h index c5d38f0b..04ecc633 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/status.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/status.h @@ -36,6 +36,8 @@ #include #include +#include + namespace google { namespace protobuf { namespace util { @@ -62,7 +64,7 @@ enum Code { }; } // namespace error -class LIBPROTOBUF_EXPORT Status { +class PROTOBUF_EXPORT Status { public: // Creates a "successful" status. Status(); @@ -88,9 +90,15 @@ class LIBPROTOBUF_EXPORT Status { int error_code() const { return error_code_; } + error::Code code() const { + return error_code_; + } StringPiece error_message() const { return error_message_; } + StringPiece message() const { + return error_message_; + } bool operator==(const Status& x) const; bool operator!=(const Status& x) const { @@ -106,11 +114,12 @@ class LIBPROTOBUF_EXPORT Status { }; // Prints a human-readable representation of 'x' to 'os'. -LIBPROTOBUF_EXPORT std::ostream& operator<<(std::ostream& os, const Status& x); - -#define EXPECT_OK(value) EXPECT_TRUE((value).ok()) +PROTOBUF_EXPORT std::ostream& operator<<(std::ostream& os, const Status& x); } // namespace util } // namespace protobuf } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_STUBS_STATUS_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/statusor.h b/third_party/protobuf-lite/google/protobuf/stubs/statusor.h index 29f869ad..c02e89a9 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/statusor.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/statusor.h @@ -33,7 +33,7 @@ // usable value, or an error Status explaining why such a value is // not present. To this end, StatusOr does not allow its Status // value to be Status::OK. Further, StatusOr does not allow the -// contained pointer to be NULL. +// contained pointer to be nullptr. // // The primary use-case for StatusOr is as the return value of a // function which may fail. @@ -89,6 +89,8 @@ #include +#include + namespace google { namespace protobuf { namespace util { @@ -114,15 +116,15 @@ class StatusOr { StatusOr(const Status& status); // NOLINT // Construct a new StatusOr with the given value. If T is a plain pointer, - // value must not be NULL. After calling this constructor, calls to + // value must not be nullptr. After calling this constructor, calls to // ValueOrDie() will succeed, and calls to status() will return OK. // // NOTE: Not explicit - we want to use StatusOr as a return type // so it is convenient and sensible to be able to do 'return T()' // when when the return type is StatusOr. // - // REQUIRES: if T is a plain pointer, value != NULL. This requirement is - // DCHECKed. In optimized builds, passing a NULL pointer here will have + // REQUIRES: if T is a plain pointer, value != nullptr. This requirement is + // DCHECKed. In optimized builds, passing a null pointer here will have // the effect of passing PosixErrorSpace::EINVAL as a fallback. StatusOr(const T& value); // NOLINT @@ -151,6 +153,7 @@ class StatusOr { // If you need to initialize a T object from the stored value, // ConsumeValueOrDie() may be more efficient. const T& ValueOrDie() const; + const T& value () const; private: Status status_; @@ -162,7 +165,7 @@ class StatusOr { namespace internal { -class LIBPROTOBUF_EXPORT StatusOrHelper { +class PROTOBUF_EXPORT StatusOrHelper { public: // Move type-agnostic error handling to the .cc. static void Crash(const util::Status& status); @@ -174,13 +177,13 @@ class LIBPROTOBUF_EXPORT StatusOrHelper { template struct StatusOrHelper::Specialize { - // For non-pointer T, a reference can never be NULL. + // For non-pointer T, a reference can never be nullptr. static inline bool IsValueNull(const T& t) { return false; } }; template struct StatusOrHelper::Specialize { - static inline bool IsValueNull(const T* t) { return t == NULL; } + static inline bool IsValueNull(const T* t) { return t == nullptr; } }; } // namespace internal @@ -202,7 +205,7 @@ inline StatusOr::StatusOr(const Status& status) { template inline StatusOr::StatusOr(const T& value) { if (internal::StatusOrHelper::Specialize::IsValueNull(value)) { - status_ = Status(error::INTERNAL, "NULL is not a vaild argument."); + status_ = Status(error::INTERNAL, "nullptr is not a valid argument."); } else { status_ = Status::OK; value_ = value; @@ -252,8 +255,18 @@ inline const T& StatusOr::ValueOrDie() const { } return value_; } + +template +inline const T& StatusOr::value() const { + if (!status_.ok()) { + internal::StatusOrHelper::Crash(status_); + } + return value_; +} } // namespace util } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_STATUSOR_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h b/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h index 9e4c82a4..aa81eb6f 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h @@ -38,26 +38,6 @@ namespace google { namespace protobuf { -// STLDeleteContainerPointers() -// For a range within a container of pointers, calls delete -// (non-array version) on these pointers. -// NOTE: for these three functions, we could just implement a DeleteObject -// functor and then call for_each() on the range and functor, but this -// requires us to pull in all of algorithm.h, which seems expensive. -// For hash_[multi]set, it is important that this deletes behind the iterator -// because the hash_set may call the hash function on the iterator when it is -// advanced, which could result in the hash function trying to deference a -// stale pointer. -template -void STLDeleteContainerPointers(ForwardIterator begin, - ForwardIterator end) { - while (begin != end) { - ForwardIterator temp = begin; - ++begin; - delete *temp; - } -} - // Inside Google, this function implements a horrible, disgusting hack in which // we reach into the string's private implementation and resize it without // initializing the new bytes. In some cases doing this can significantly @@ -82,37 +62,7 @@ inline void STLStringResizeUninitialized(string* s, size_t new_size) { // already work on all current implementations. inline char* string_as_array(string* str) { // DO NOT USE const_cast(str->data())! See the unittest for why. - return str->empty() ? NULL : &*str->begin(); -} - -// STLDeleteElements() deletes all the elements in an STL container and clears -// the container. This function is suitable for use with a vector, set, -// hash_set, or any other STL container which defines sensible begin(), end(), -// and clear() methods. -// -// If container is NULL, this function is a no-op. -// -// As an alternative to calling STLDeleteElements() directly, consider -// ElementDeleter (defined below), which ensures that your container's elements -// are deleted when the ElementDeleter goes out of scope. -template -void STLDeleteElements(T *container) { - if (!container) return; - STLDeleteContainerPointers(container->begin(), container->end()); - container->clear(); -} - -// Given an STL container consisting of (key, value) pairs, STLDeleteValues -// deletes all the "value" components and clears the container. Does nothing -// in the case it's given a NULL pointer. - -template -void STLDeleteValues(T *v) { - if (!v) return; - for (typename T::iterator i = v->begin(); i != v->end(); ++i) { - delete i->second; - } - v->clear(); + return str->empty() ? nullptr : &*str->begin(); } } // namespace protobuf diff --git a/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h b/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h index 563ff75d..2ae38813 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h @@ -76,30 +76,30 @@ // // There are several ways to create a null StringPiece: // StringPiece() -// StringPiece(NULL) -// StringPiece(NULL, 0) -// For all of the above, sp.data() == NULL, sp.length() == 0, +// StringPiece(nullptr) +// StringPiece(nullptr, 0) +// For all of the above, sp.data() == nullptr, sp.length() == 0, // and sp.empty() == true. Also, if you create a StringPiece with -// a non-NULL pointer then sp.data() != NULL. Once created, -// sp.data() will stay either NULL or not-NULL, except if you call +// a non-null pointer then sp.data() != nullptr. Once created, +// sp.data() will stay either nullptr or not-nullptr, except if you call // sp.clear() or sp.set(). // -// Thus, you can use StringPiece(NULL) to signal an out-of-band value +// Thus, you can use StringPiece(nullptr) to signal an out-of-band value // that is different from other StringPiece values. This is similar -// to the way that const char* p1 = NULL; is different from +// to the way that const char* p1 = nullptr; is different from // const char* p2 = "";. // // There are many ways to create an empty StringPiece: // StringPiece() -// StringPiece(NULL) -// StringPiece(NULL, 0) +// StringPiece(nullptr) +// StringPiece(nullptr, 0) // StringPiece("") // StringPiece("", 0) // StringPiece("abcdef", 0) // StringPiece("abcdef"+6, 0) // For all of the above, sp.length() will be 0 and sp.empty() will be true. -// For some empty StringPiece values, sp.data() will be NULL. -// For some empty StringPiece values, sp.data() will not be NULL. +// For some empty StringPiece values, sp.data() will be nullptr. +// For some empty StringPiece values, sp.data() will not be nullptr. // // Be careful not to confuse: null StringPiece and empty StringPiece. // The set of empty StringPieces properly includes the set of null StringPieces. @@ -109,20 +109,20 @@ // All empty StringPiece values compare equal to each other. // Even a null StringPieces compares equal to a non-null empty StringPiece: // StringPiece() == StringPiece("", 0) -// StringPiece(NULL) == StringPiece("abc", 0) -// StringPiece(NULL, 0) == StringPiece("abcdef"+6, 0) +// StringPiece(nullptr) == StringPiece("abc", 0) +// StringPiece(nullptr, 0) == StringPiece("abcdef"+6, 0) // // Look carefully at this example: -// StringPiece("") == NULL +// StringPiece("") == nullptr // True or false? TRUE, because StringPiece::operator== converts -// the right-hand side from NULL to StringPiece(NULL), +// the right-hand side from nullptr to StringPiece(nullptr), // and then compares two zero-length spans of characters. // However, we are working to make this example produce a compile error. // // Suppose you want to write: -// bool TestWhat?(StringPiece sp) { return sp == NULL; } // BAD +// bool TestWhat?(StringPiece sp) { return sp == nullptr; } // BAD // Do not do that. Write one of these instead: -// bool TestNull(StringPiece sp) { return sp.data() == NULL; } +// bool TestNull(StringPiece sp) { return sp.data() == nullptr; } // bool TestEmpty(StringPiece sp) { return sp.empty(); } // The intent of TestWhat? is unclear. Did you mean TestNull or TestEmpty? // Right now, TestWhat? behaves likes TestEmpty. @@ -151,6 +151,8 @@ #include #include +#include + namespace google { namespace protobuf { // StringPiece has *two* size types. @@ -175,7 +177,7 @@ typedef string::difference_type stringpiece_ssize_type; #define STRINGPIECE_CHECK_SIZE 0 #endif -class LIBPROTOBUF_EXPORT StringPiece { +class PROTOBUF_EXPORT StringPiece { private: const char* ptr_; stringpiece_ssize_type length_; @@ -207,11 +209,11 @@ class LIBPROTOBUF_EXPORT StringPiece { // // Style guide exception granted: // http://goto/style-guide-exception-20978288 - StringPiece() : ptr_(NULL), length_(0) {} + StringPiece() : ptr_(nullptr), length_(0) {} StringPiece(const char* str) // NOLINT(runtime/explicit) : ptr_(str), length_(0) { - if (str != NULL) { + if (str != nullptr) { length_ = CheckedSsizeTFromSizeT(strlen(str)); } } @@ -248,7 +250,7 @@ class LIBPROTOBUF_EXPORT StringPiece { bool empty() const { return length_ == 0; } void clear() { - ptr_ = NULL; + ptr_ = nullptr; length_ = 0; } @@ -260,7 +262,7 @@ class LIBPROTOBUF_EXPORT StringPiece { void set(const char* str) { ptr_ = str; - if (str != NULL) + if (str != nullptr) length_ = CheckedSsizeTFromSizeT(strlen(str)); else length_ = 0; @@ -309,7 +311,7 @@ class LIBPROTOBUF_EXPORT StringPiece { // for a StringPiece be called "as_string()". We also leave the // "as_string()" method defined here for existing code. string ToString() const { - if (ptr_ == NULL) return string(); + if (ptr_ == nullptr) return string(); return string(data(), static_cast(size())); } @@ -463,6 +465,9 @@ struct StringPiecePod { std::string ToString() const { return std::string(data_, static_cast(size_)); } + + operator string() const { return ToString(); } + private: const char* data_; stringpiece_ssize_type size_; @@ -476,7 +481,7 @@ GOOGLE_PROTOBUF_HASH_NAMESPACE_DECLARATION_START template<> struct hash { size_t operator()(const StringPiece& s) const { size_t result = 0; - for (const char *str = s.data(), *end = str + s.size(); str < end; str++) { + for (const char *str = s.data(), *end = str + s.size(); str < end; str++) { result = 5 * result + static_cast(*str); } return result; @@ -484,4 +489,6 @@ template<> struct hash { }; GOOGLE_PROTOBUF_HASH_NAMESPACE_DECLARATION_END +#include + #endif // STRINGS_STRINGPIECE_H_ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h b/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h index 7183ec6a..253d736a 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h @@ -46,31 +46,38 @@ #include +#include + namespace google { namespace protobuf { // Return a C++ string -LIBPROTOBUF_EXPORT extern string StringPrintf(const char* format, ...); +PROTOBUF_EXPORT extern string StringPrintf(const char* format, ...); // Store result into a supplied string and return it -LIBPROTOBUF_EXPORT extern const string& SStringPrintf(string* dst, const char* format, ...); +PROTOBUF_EXPORT extern const string& SStringPrintf(string* dst, + const char* format, ...); // Append result to a supplied string -LIBPROTOBUF_EXPORT extern void StringAppendF(string* dst, const char* format, ...); +PROTOBUF_EXPORT extern void StringAppendF(string* dst, const char* format, ...); // Lower-level routine that takes a va_list and appends to a specified // string. All other routines are just convenience wrappers around it. -LIBPROTOBUF_EXPORT extern void StringAppendV(string* dst, const char* format, va_list ap); +PROTOBUF_EXPORT extern void StringAppendV(string* dst, const char* format, + va_list ap); // The max arguments supported by StringPrintfVector -LIBPROTOBUF_EXPORT extern const int kStringPrintfVectorMaxArgs; +PROTOBUF_EXPORT extern const int kStringPrintfVectorMaxArgs; // You can use this version when all your arguments are strings, but // you don't know how many arguments you'll have at compile time. // StringPrintfVector will LOG(FATAL) if v.size() > kStringPrintfVectorMaxArgs -LIBPROTOBUF_EXPORT extern string StringPrintfVector(const char* format, const std::vector& v); +PROTOBUF_EXPORT extern string StringPrintfVector(const char* format, + const std::vector& v); } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_STRINGPRINTF_H diff --git a/third_party/protobuf-lite/google/protobuf/stubs/strutil.h b/third_party/protobuf-lite/google/protobuf/stubs/strutil.h index a839b8b3..79a7aeb5 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/strutil.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/strutil.h @@ -33,15 +33,18 @@ #ifndef GOOGLE_PROTOBUF_STUBS_STRUTIL_H__ #define GOOGLE_PROTOBUF_STUBS_STRUTIL_H__ -#include -#include #include #include +#include + +#include +#include +#include namespace google { namespace protobuf { -#ifdef _MSC_VER +#if defined(_MSC_VER) && _MSC_VER < 1800 #define strtoll _strtoi64 #define strtoull _strtoui64 #elif defined(__DECCXX) && defined(__osf__) @@ -116,6 +119,11 @@ inline bool HasPrefixString(const string& str, str.compare(0, prefix.size(), prefix) == 0; } +inline bool HasPrefixString(StringPiece str, StringPiece prefix) { + return str.size() >= prefix.size() && + memcmp(str.data(), prefix.data(), prefix.size()) == 0; +} + inline string StripPrefixString(const string& str, const string& prefix) { if (HasPrefixString(str, prefix)) { return str.substr(prefix.size()); @@ -155,13 +163,10 @@ inline string StripSuffixString(const string& str, const string& suffix) { // StripWhitespace // Removes whitespaces from both ends of the given string. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT void ReplaceCharacters(string* s, const char* remove, - char replacewith); -LIBPROTOBUF_EXPORT void StripString(string* s, const char* remove, - char replacewith); - -LIBPROTOBUF_EXPORT void StripWhitespace(string* s); +PROTOBUF_EXPORT void ReplaceCharacters(string* s, const char* remove, + char replacewith); +PROTOBUF_EXPORT void StripWhitespace(string* s); // ---------------------------------------------------------------------- // LowerString() @@ -203,8 +208,8 @@ inline string ToUpper(const string& s) { // happened or not. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT string StringReplace(const string& s, const string& oldsub, - const string& newsub, bool replace_all); +PROTOBUF_EXPORT string StringReplace(const string& s, const string& oldsub, + const string& newsub, bool replace_all); // ---------------------------------------------------------------------- // SplitStringUsing() @@ -212,8 +217,8 @@ LIBPROTOBUF_EXPORT string StringReplace(const string& s, const string& oldsub, // to 'result'. If there are consecutive delimiters, this function skips // over all of them. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT void SplitStringUsing(const string& full, const char* delim, - std::vector* res); +PROTOBUF_EXPORT void SplitStringUsing(const string& full, const char* delim, + std::vector* res); // Split a string using one or more byte delimiters, presented // as a nul-terminated c string. Append the components to 'result'. @@ -223,9 +228,9 @@ LIBPROTOBUF_EXPORT void SplitStringUsing(const string& full, const char* delim, // // If "full" is the empty string, yields an empty string as the only value. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT void SplitStringAllowEmpty(const string& full, - const char* delim, - std::vector* result); +PROTOBUF_EXPORT void SplitStringAllowEmpty(const string& full, + const char* delim, + std::vector* result); // ---------------------------------------------------------------------- // Split() @@ -250,8 +255,8 @@ inline std::vector Split( // another takes a pointer to the target string. In the latter case the // target string is cleared and overwritten. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT void JoinStrings(const std::vector& components, - const char* delim, string* result); +PROTOBUF_EXPORT void JoinStrings(const std::vector& components, + const char* delim, string* result); inline string JoinStrings(const std::vector& components, const char* delim) { @@ -285,15 +290,15 @@ inline string JoinStrings(const std::vector& components, // // Errors: In the first form of the call, errors are reported with // LOG(ERROR). The same is true for the second form of the call if -// the pointer to the string std::vector is NULL; otherwise, error +// the pointer to the string std::vector is nullptr; otherwise, error // messages are stored in the std::vector. In either case, the effect on // the dest array is not defined, but rest of the source will be // processed. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest); -LIBPROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest, - std::vector *errors); +PROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest); +PROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest, + std::vector* errors); // ---------------------------------------------------------------------- // UnescapeCEscapeString() @@ -304,16 +309,16 @@ LIBPROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest, // to be the same. // // The second call stores its errors in a supplied string vector. -// If the string vector pointer is NULL, it reports the errors with LOG(). +// If the string vector pointer is nullptr, it reports the errors with LOG(). // // In the first and second calls, the length of dest is returned. In the // the third call, the new string is returned. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int UnescapeCEscapeString(const string& src, string* dest); -LIBPROTOBUF_EXPORT int UnescapeCEscapeString(const string& src, string* dest, - std::vector *errors); -LIBPROTOBUF_EXPORT string UnescapeCEscapeString(const string& src); +PROTOBUF_EXPORT int UnescapeCEscapeString(const string& src, string* dest); +PROTOBUF_EXPORT int UnescapeCEscapeString(const string& src, string* dest, + std::vector* errors); +PROTOBUF_EXPORT string UnescapeCEscapeString(const string& src); // ---------------------------------------------------------------------- // CEscape() @@ -322,21 +327,21 @@ LIBPROTOBUF_EXPORT string UnescapeCEscapeString(const string& src); // // Escaped chars: \n, \r, \t, ", ', \, and !isprint(). // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT string CEscape(const string& src); +PROTOBUF_EXPORT string CEscape(const string& src); // ---------------------------------------------------------------------- // CEscapeAndAppend() // Escapes 'src' using C-style escape sequences, and appends the escaped // string to 'dest'. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT void CEscapeAndAppend(StringPiece src, string* dest); +PROTOBUF_EXPORT void CEscapeAndAppend(StringPiece src, string* dest); namespace strings { // Like CEscape() but does not escape bytes with the upper bit set. -LIBPROTOBUF_EXPORT string Utf8SafeCEscape(const string& src); +PROTOBUF_EXPORT string Utf8SafeCEscape(const string& src); // Like CEscape() but uses hex (\x) escapes instead of octals. -LIBPROTOBUF_EXPORT string CHexEscape(const string& src); +PROTOBUF_EXPORT string CHexEscape(const string& src); } // namespace strings // ---------------------------------------------------------------------- @@ -349,10 +354,10 @@ LIBPROTOBUF_EXPORT string CHexEscape(const string& src); // platforms, so using these is safer, from the point of view of // overflow behavior, than using the standard libc functions. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int32 strto32_adaptor(const char *nptr, char **endptr, - int base); -LIBPROTOBUF_EXPORT uint32 strtou32_adaptor(const char *nptr, char **endptr, - int base); +PROTOBUF_EXPORT int32 strto32_adaptor(const char* nptr, char** endptr, + int base); +PROTOBUF_EXPORT uint32 strtou32_adaptor(const char* nptr, char** endptr, + int base); inline int32 strto32(const char *nptr, char **endptr, int base) { if (sizeof(int32) == sizeof(long)) @@ -391,10 +396,10 @@ inline uint64 strtou64(const char *nptr, char **endptr, int base) { // safe_strtof() // safe_strtod() // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT bool safe_strtob(StringPiece str, bool* value); +PROTOBUF_EXPORT bool safe_strtob(StringPiece str, bool* value); -LIBPROTOBUF_EXPORT bool safe_strto32(const string& str, int32* value); -LIBPROTOBUF_EXPORT bool safe_strtou32(const string& str, uint32* value); +PROTOBUF_EXPORT bool safe_strto32(const string& str, int32* value); +PROTOBUF_EXPORT bool safe_strtou32(const string& str, uint32* value); inline bool safe_strto32(const char* str, int32* value) { return safe_strto32(string(str), value); } @@ -408,8 +413,8 @@ inline bool safe_strtou32(StringPiece str, uint32* value) { return safe_strtou32(str.ToString(), value); } -LIBPROTOBUF_EXPORT bool safe_strto64(const string& str, int64* value); -LIBPROTOBUF_EXPORT bool safe_strtou64(const string& str, uint64* value); +PROTOBUF_EXPORT bool safe_strto64(const string& str, int64* value); +PROTOBUF_EXPORT bool safe_strtou64(const string& str, uint64* value); inline bool safe_strto64(const char* str, int64* value) { return safe_strto64(string(str), value); } @@ -423,8 +428,8 @@ inline bool safe_strtou64(StringPiece str, uint64* value) { return safe_strtou64(str.ToString(), value); } -LIBPROTOBUF_EXPORT bool safe_strtof(const char* str, float* value); -LIBPROTOBUF_EXPORT bool safe_strtod(const char* str, double* value); +PROTOBUF_EXPORT bool safe_strtof(const char* str, float* value); +PROTOBUF_EXPORT bool safe_strtod(const char* str, double* value); inline bool safe_strtof(const string& str, float* value) { return safe_strtof(str.c_str(), value); } @@ -464,13 +469,13 @@ inline bool safe_strtod(StringPiece str, double* value) { // DoubleToBuffer() and FloatToBuffer(). static const int kFastToBufferSize = 32; -LIBPROTOBUF_EXPORT char* FastInt32ToBuffer(int32 i, char* buffer); -LIBPROTOBUF_EXPORT char* FastInt64ToBuffer(int64 i, char* buffer); +PROTOBUF_EXPORT char* FastInt32ToBuffer(int32 i, char* buffer); +PROTOBUF_EXPORT char* FastInt64ToBuffer(int64 i, char* buffer); char* FastUInt32ToBuffer(uint32 i, char* buffer); // inline below char* FastUInt64ToBuffer(uint64 i, char* buffer); // inline below -LIBPROTOBUF_EXPORT char* FastHexToBuffer(int i, char* buffer); -LIBPROTOBUF_EXPORT char* FastHex64ToBuffer(uint64 i, char* buffer); -LIBPROTOBUF_EXPORT char* FastHex32ToBuffer(uint32 i, char* buffer); +PROTOBUF_EXPORT char* FastHexToBuffer(int i, char* buffer); +PROTOBUF_EXPORT char* FastHex64ToBuffer(uint64 i, char* buffer); +PROTOBUF_EXPORT char* FastHex32ToBuffer(uint32 i, char* buffer); // at least 22 bytes long inline char* FastIntToBuffer(int i, char* buffer) { @@ -506,10 +511,10 @@ inline char* FastULongToBuffer(unsigned long i, char* buffer) { // terminating the string). // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT char* FastInt32ToBufferLeft(int32 i, char* buffer); -LIBPROTOBUF_EXPORT char* FastUInt32ToBufferLeft(uint32 i, char* buffer); -LIBPROTOBUF_EXPORT char* FastInt64ToBufferLeft(int64 i, char* buffer); -LIBPROTOBUF_EXPORT char* FastUInt64ToBufferLeft(uint64 i, char* buffer); +PROTOBUF_EXPORT char* FastInt32ToBufferLeft(int32 i, char* buffer); +PROTOBUF_EXPORT char* FastUInt32ToBufferLeft(uint32 i, char* buffer); +PROTOBUF_EXPORT char* FastInt64ToBufferLeft(int64 i, char* buffer); +PROTOBUF_EXPORT char* FastUInt64ToBufferLeft(uint64 i, char* buffer); // Just define these in terms of the above. inline char* FastUInt32ToBuffer(uint32 i, char* buffer) { @@ -531,12 +536,12 @@ inline string SimpleBtoa(bool value) { // // Return value: string // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT string SimpleItoa(int i); -LIBPROTOBUF_EXPORT string SimpleItoa(unsigned int i); -LIBPROTOBUF_EXPORT string SimpleItoa(long i); -LIBPROTOBUF_EXPORT string SimpleItoa(unsigned long i); -LIBPROTOBUF_EXPORT string SimpleItoa(long long i); -LIBPROTOBUF_EXPORT string SimpleItoa(unsigned long long i); +PROTOBUF_EXPORT string SimpleItoa(int i); +PROTOBUF_EXPORT string SimpleItoa(unsigned int i); +PROTOBUF_EXPORT string SimpleItoa(long i); +PROTOBUF_EXPORT string SimpleItoa(unsigned long i); +PROTOBUF_EXPORT string SimpleItoa(long long i); +PROTOBUF_EXPORT string SimpleItoa(unsigned long long i); // ---------------------------------------------------------------------- // SimpleDtoa() @@ -557,11 +562,11 @@ LIBPROTOBUF_EXPORT string SimpleItoa(unsigned long long i); // // Return value: string // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT string SimpleDtoa(double value); -LIBPROTOBUF_EXPORT string SimpleFtoa(float value); +PROTOBUF_EXPORT string SimpleDtoa(double value); +PROTOBUF_EXPORT string SimpleFtoa(float value); -LIBPROTOBUF_EXPORT char* DoubleToBuffer(double i, char* buffer); -LIBPROTOBUF_EXPORT char* FloatToBuffer(float i, char* buffer); +PROTOBUF_EXPORT char* DoubleToBuffer(double i, char* buffer); +PROTOBUF_EXPORT char* FloatToBuffer(float i, char* buffer); // In practice, doubles should never need more than 24 bytes and floats // should never need more than 14 (including null terminators), but we @@ -610,7 +615,7 @@ struct Hex { } }; -struct LIBPROTOBUF_EXPORT AlphaNum { +struct PROTOBUF_EXPORT AlphaNum { const char *piece_data_; // move these to string_ref eventually size_t piece_size_; // move these to string_ref eventually @@ -619,16 +624,26 @@ struct LIBPROTOBUF_EXPORT AlphaNum { // No bool ctor -- bools convert to an integral type. // A bool ctor would also convert incoming pointers (bletch). - AlphaNum(int32 i32) + AlphaNum(int i32) : piece_data_(digits), piece_size_(FastInt32ToBufferLeft(i32, digits) - &digits[0]) {} - AlphaNum(uint32 u32) + AlphaNum(unsigned int u32) : piece_data_(digits), piece_size_(FastUInt32ToBufferLeft(u32, digits) - &digits[0]) {} - AlphaNum(int64 i64) + AlphaNum(long long i64) : piece_data_(digits), piece_size_(FastInt64ToBufferLeft(i64, digits) - &digits[0]) {} - AlphaNum(uint64 u64) + AlphaNum(unsigned long long u64) + : piece_data_(digits), + piece_size_(FastUInt64ToBufferLeft(u64, digits) - &digits[0]) {} + + // Note: on some architectures, "long" is only 32 bits, not 64, but the + // performance hit of using FastInt64ToBufferLeft to handle 32-bit values + // is quite minor. + AlphaNum(long i64) + : piece_data_(digits), + piece_size_(FastInt64ToBufferLeft(i64, digits) - &digits[0]) {} + AlphaNum(unsigned long u64) : piece_data_(digits), piece_size_(FastUInt64ToBufferLeft(u64, digits) - &digits[0]) {} @@ -692,30 +707,30 @@ using strings::AlphaNum; // be a reference into str. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b); -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c); -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d); -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d, - const AlphaNum& e); -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d, - const AlphaNum& e, const AlphaNum& f); -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d, - const AlphaNum& e, const AlphaNum& f, - const AlphaNum& g); -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d, - const AlphaNum& e, const AlphaNum& f, - const AlphaNum& g, const AlphaNum& h); -LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d, - const AlphaNum& e, const AlphaNum& f, - const AlphaNum& g, const AlphaNum& h, - const AlphaNum& i); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f, + const AlphaNum& g); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f, + const AlphaNum& g, const AlphaNum& h); +PROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f, + const AlphaNum& g, const AlphaNum& h, + const AlphaNum& i); inline string StrCat(const AlphaNum& a) { return string(a.data(), a.size()); } @@ -740,14 +755,14 @@ inline string StrCat(const AlphaNum& a) { return string(a.data(), a.size()); } // worked around as consecutive calls to StrAppend are quite efficient. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a); -LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, - const AlphaNum& b); -LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, - const AlphaNum& b, const AlphaNum& c); -LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, - const AlphaNum& b, const AlphaNum& c, - const AlphaNum& d); +PROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a); +PROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, + const AlphaNum& b); +PROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c); +PROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c, + const AlphaNum& d); // ---------------------------------------------------------------------- // Join() @@ -777,7 +792,7 @@ string Join(const Range& components, // ToHex() // Return a lower-case hex string representation of the given integer. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT string ToHex(uint64 num); +PROTOBUF_EXPORT string ToHex(uint64 num); // ---------------------------------------------------------------------- // GlobalReplaceSubstring() @@ -786,9 +801,9 @@ LIBPROTOBUF_EXPORT string ToHex(uint64 num); // // NOTE: The string pieces must not overlap s. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int GlobalReplaceSubstring(const string& substring, - const string& replacement, - string* s); +PROTOBUF_EXPORT int GlobalReplaceSubstring(const string& substring, + const string& replacement, + string* s); // ---------------------------------------------------------------------- // Base64Unescape() @@ -796,7 +811,7 @@ LIBPROTOBUF_EXPORT int GlobalReplaceSubstring(const string& substring, // writes it to "dest". If src contains invalid characters, dest is cleared // and the function returns false. Returns true on success. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT bool Base64Unescape(StringPiece src, string* dest); +PROTOBUF_EXPORT bool Base64Unescape(StringPiece src, string* dest); // ---------------------------------------------------------------------- // WebSafeBase64Unescape() @@ -809,18 +824,17 @@ LIBPROTOBUF_EXPORT bool Base64Unescape(StringPiece src, string* dest); // returns false (with dest empty) if src contains invalid chars; for // this version src and dest must be different strings. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int WebSafeBase64Unescape(const char* src, int slen, - char* dest, int szdest); -LIBPROTOBUF_EXPORT bool WebSafeBase64Unescape(StringPiece src, string* dest); +PROTOBUF_EXPORT int WebSafeBase64Unescape(const char* src, int slen, char* dest, + int szdest); +PROTOBUF_EXPORT bool WebSafeBase64Unescape(StringPiece src, string* dest); // Return the length to use for the output buffer given to the base64 escape // routines. Make sure to use the same value for do_padding in both. // This function may return incorrect results if given input_len values that // are extremely high, which should happen rarely. -LIBPROTOBUF_EXPORT int CalculateBase64EscapedLen(int input_len, - bool do_padding); +PROTOBUF_EXPORT int CalculateBase64EscapedLen(int input_len, bool do_padding); // Use this version when calling Base64Escape without a do_padding arg. -LIBPROTOBUF_EXPORT int CalculateBase64EscapedLen(int input_len); +PROTOBUF_EXPORT int CalculateBase64EscapedLen(int input_len); // ---------------------------------------------------------------------- // Base64Escape() @@ -834,23 +848,23 @@ LIBPROTOBUF_EXPORT int CalculateBase64EscapedLen(int input_len); // to escape them. It also has an extra parameter "do_padding", // which when set to false will prevent padding with "=". // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int Base64Escape(const unsigned char* src, int slen, - char* dest, int szdest); -LIBPROTOBUF_EXPORT int WebSafeBase64Escape( - const unsigned char* src, int slen, char* dest, - int szdest, bool do_padding); +PROTOBUF_EXPORT int Base64Escape(const unsigned char* src, int slen, char* dest, + int szdest); +PROTOBUF_EXPORT int WebSafeBase64Escape(const unsigned char* src, int slen, + char* dest, int szdest, + bool do_padding); // Encode src into dest with padding. -LIBPROTOBUF_EXPORT void Base64Escape(StringPiece src, string* dest); +PROTOBUF_EXPORT void Base64Escape(StringPiece src, string* dest); // Encode src into dest web-safely without padding. -LIBPROTOBUF_EXPORT void WebSafeBase64Escape(StringPiece src, string* dest); +PROTOBUF_EXPORT void WebSafeBase64Escape(StringPiece src, string* dest); // Encode src into dest web-safely with padding. -LIBPROTOBUF_EXPORT void WebSafeBase64EscapeWithPadding(StringPiece src, - string* dest); +PROTOBUF_EXPORT void WebSafeBase64EscapeWithPadding(StringPiece src, + string* dest); -LIBPROTOBUF_EXPORT void Base64Escape(const unsigned char* src, int szsrc, - string* dest, bool do_padding); -LIBPROTOBUF_EXPORT void WebSafeBase64Escape(const unsigned char* src, int szsrc, - string* dest, bool do_padding); +PROTOBUF_EXPORT void Base64Escape(const unsigned char* src, int szsrc, + string* dest, bool do_padding); +PROTOBUF_EXPORT void WebSafeBase64Escape(const unsigned char* src, int szsrc, + string* dest, bool do_padding); inline bool IsValidCodePoint(uint32 code_point) { return code_point < 0xD800 || @@ -864,15 +878,75 @@ static const int UTFmax = 4; // in any external dependencies. The output buffer must be as least 4 bytes // large. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int EncodeAsUTF8Char(uint32 code_point, char* output); +PROTOBUF_EXPORT int EncodeAsUTF8Char(uint32 code_point, char* output); // ---------------------------------------------------------------------- // UTF8FirstLetterNumBytes() // Length of the first UTF-8 character. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int UTF8FirstLetterNumBytes(const char* src, int len); +PROTOBUF_EXPORT int UTF8FirstLetterNumBytes(const char* src, int len); + +// From google3/third_party/absl/strings/escaping.h + +// ---------------------------------------------------------------------- +// CleanStringLineEndings() +// Clean up a multi-line string to conform to Unix line endings. +// Reads from src and appends to dst, so usually dst should be empty. +// +// If there is no line ending at the end of a non-empty string, it can +// be added automatically. +// +// Four different types of input are correctly handled: +// +// - Unix/Linux files: line ending is LF: pass through unchanged +// +// - DOS/Windows files: line ending is CRLF: convert to LF +// +// - Legacy Mac files: line ending is CR: convert to LF +// +// - Garbled files: random line endings: convert gracefully +// lonely CR, lonely LF, CRLF: convert to LF +// +// @param src The multi-line string to convert +// @param dst The converted string is appended to this string +// @param auto_end_last_line Automatically terminate the last line +// +// Limitations: +// +// This does not do the right thing for CRCRLF files created by +// broken programs that do another Unix->DOS conversion on files +// that are already in CRLF format. For this, a two-pass approach +// brute-force would be needed that +// +// (1) determines the presence of LF (first one is ok) +// (2) if yes, removes any CR, else convert every CR to LF +PROTOBUF_EXPORT void CleanStringLineEndings(const string& src, string* dst, + bool auto_end_last_line); + +// Same as above, but transforms the argument in place. +PROTOBUF_EXPORT void CleanStringLineEndings(string* str, + bool auto_end_last_line); + +namespace strings { +inline bool EndsWith(StringPiece text, StringPiece suffix) { + return suffix.empty() || + (text.size() >= suffix.size() && + memcmp(text.data() + (text.size() - suffix.size()), suffix.data(), + suffix.size()) == 0); +} +} // namespace strings + +namespace internal { + +// A locale-independent version of the standard strtod(), which always +// uses a dot as the decimal separator. +double NoLocaleStrtod(const char* str, char** endptr); + +} // namespace internal } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_STRUTIL_H__ diff --git a/third_party/protobuf-lite/google/protobuf/stubs/time.h b/third_party/protobuf-lite/google/protobuf/stubs/time.h index 45607ca9..b52f3f96 100644 --- a/third_party/protobuf-lite/google/protobuf/stubs/time.h +++ b/third_party/protobuf-lite/google/protobuf/stubs/time.h @@ -32,6 +32,8 @@ #include +#include + namespace google { namespace protobuf { namespace internal { @@ -49,27 +51,30 @@ struct DateTime { // negative to represent time before 1970-01-01) to DateTime. Returns false // if the timestamp is not in the range between 0001-01-01T00:00:00 and // 9999-12-31T23:59:59. -bool LIBPROTOBUF_EXPORT SecondsToDateTime(int64 seconds, DateTime* time); +bool PROTOBUF_EXPORT SecondsToDateTime(int64 seconds, DateTime* time); // Converts DateTime to a timestamp (seconds since 1970-01-01T00:00:00). // Returns false if the DateTime is not valid or is not in the valid range. -bool LIBPROTOBUF_EXPORT DateTimeToSeconds(const DateTime& time, int64* seconds); +bool PROTOBUF_EXPORT DateTimeToSeconds(const DateTime& time, int64* seconds); -void LIBPROTOBUF_EXPORT GetCurrentTime(int64* seconds, int32* nanos); +void PROTOBUF_EXPORT GetCurrentTime(int64* seconds, int32* nanos); -// Formats a time string in RFC3339 fromat. +// Formats a time string in RFC3339 format. // // For example, "2015-05-20T13:29:35.120Z". For nanos, 0, 3, 6 or 9 fractional // digits will be used depending on how many are required to represent the exact // value. // // Note that "nanos" must in the range of [0, 999999999]. -string LIBPROTOBUF_EXPORT FormatTime(int64 seconds, int32 nanos); +string PROTOBUF_EXPORT FormatTime(int64 seconds, int32 nanos); // Parses a time string. This method accepts RFC3339 date/time string with UTC // offset. For example, "2015-05-20T13:29:35.120-08:00". -bool LIBPROTOBUF_EXPORT ParseTime(const string& value, int64* seconds, int32* nanos); +bool PROTOBUF_EXPORT ParseTime(const string& value, int64* seconds, + int32* nanos); } // namespace internal } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_STUBS_TIME_H_ diff --git a/third_party/protobuf-lite/google/protobuf/wire_format_lite.h b/third_party/protobuf-lite/google/protobuf/wire_format_lite.h index 77eaa9a6..c742fe86 100644 --- a/third_party/protobuf-lite/google/protobuf/wire_format_lite.h +++ b/third_party/protobuf-lite/google/protobuf/wire_format_lite.h @@ -43,10 +43,13 @@ #include #include +#include #include +#include #include -#include +#include #include +#include // Do UTF-8 validation on string type in Debug build only #ifndef NDEBUG @@ -55,27 +58,23 @@ // Avoid conflict with iOS where #defines TYPE_BOOL. // -// If some one needs the macro TYPE_BOOL in a file that includes this header, it's -// possible to bring it back using push/pop_macro as follows. +// If some one needs the macro TYPE_BOOL in a file that includes this header, +// it's possible to bring it back using push/pop_macro as follows. // // #pragma push_macro("TYPE_BOOL") // #include this header and/or all headers that need the macro to be undefined. // #pragma pop_macro("TYPE_BOOL") #undef TYPE_BOOL -namespace google { - -namespace protobuf { - template class RepeatedField; // repeated_field.h -} +namespace google { namespace protobuf { namespace internal { -class StringPieceField; +#include // This class is for internal use by the protocol buffer library and by -// protocol-complier-generated message classes. It must not be called +// protocol-compiler-generated message classes. It must not be called // directly by clients. // // This class contains helpers for implementing the binary protocol buffer @@ -83,9 +82,8 @@ class StringPieceField; // reflection. // // This class is really a namespace that contains only static methods. -class LIBPROTOBUF_EXPORT WireFormatLite { +class PROTOBUF_EXPORT WireFormatLite { public: - // ----------------------------------------------------------------- // Helper constants and functions related to the format. These are // mostly meant for internal and generated code to use. @@ -102,50 +100,50 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // unrecognized fields for forwards compatibility. enum WireType { - WIRETYPE_VARINT = 0, - WIRETYPE_FIXED64 = 1, + WIRETYPE_VARINT = 0, + WIRETYPE_FIXED64 = 1, WIRETYPE_LENGTH_DELIMITED = 2, - WIRETYPE_START_GROUP = 3, - WIRETYPE_END_GROUP = 4, - WIRETYPE_FIXED32 = 5, + WIRETYPE_START_GROUP = 3, + WIRETYPE_END_GROUP = 4, + WIRETYPE_FIXED32 = 5, }; // Lite alternative to FieldDescriptor::Type. Must be kept in sync. enum FieldType { - TYPE_DOUBLE = 1, - TYPE_FLOAT = 2, - TYPE_INT64 = 3, - TYPE_UINT64 = 4, - TYPE_INT32 = 5, - TYPE_FIXED64 = 6, - TYPE_FIXED32 = 7, - TYPE_BOOL = 8, - TYPE_STRING = 9, - TYPE_GROUP = 10, - TYPE_MESSAGE = 11, - TYPE_BYTES = 12, - TYPE_UINT32 = 13, - TYPE_ENUM = 14, - TYPE_SFIXED32 = 15, - TYPE_SFIXED64 = 16, - TYPE_SINT32 = 17, - TYPE_SINT64 = 18, - MAX_FIELD_TYPE = 18, + TYPE_DOUBLE = 1, + TYPE_FLOAT = 2, + TYPE_INT64 = 3, + TYPE_UINT64 = 4, + TYPE_INT32 = 5, + TYPE_FIXED64 = 6, + TYPE_FIXED32 = 7, + TYPE_BOOL = 8, + TYPE_STRING = 9, + TYPE_GROUP = 10, + TYPE_MESSAGE = 11, + TYPE_BYTES = 12, + TYPE_UINT32 = 13, + TYPE_ENUM = 14, + TYPE_SFIXED32 = 15, + TYPE_SFIXED64 = 16, + TYPE_SINT32 = 17, + TYPE_SINT64 = 18, + MAX_FIELD_TYPE = 18, }; // Lite alternative to FieldDescriptor::CppType. Must be kept in sync. enum CppType { - CPPTYPE_INT32 = 1, - CPPTYPE_INT64 = 2, - CPPTYPE_UINT32 = 3, - CPPTYPE_UINT64 = 4, - CPPTYPE_DOUBLE = 5, - CPPTYPE_FLOAT = 6, - CPPTYPE_BOOL = 7, - CPPTYPE_ENUM = 8, - CPPTYPE_STRING = 9, - CPPTYPE_MESSAGE = 10, - MAX_CPPTYPE = 10, + CPPTYPE_INT32 = 1, + CPPTYPE_INT64 = 2, + CPPTYPE_UINT32 = 3, + CPPTYPE_UINT64 = 4, + CPPTYPE_DOUBLE = 5, + CPPTYPE_FLOAT = 6, + CPPTYPE_BOOL = 7, + CPPTYPE_ENUM = 8, + CPPTYPE_STRING = 9, + CPPTYPE_MESSAGE = 10, + MAX_CPPTYPE = 10, }; // Helper method to get the CppType for a particular Type. @@ -158,16 +156,16 @@ class LIBPROTOBUF_EXPORT WireFormatLite { } // Number of bits in a tag which identify the wire type. - static const int kTagTypeBits = 3; + static constexpr int kTagTypeBits = 3; // Mask for those bits. - static const uint32 kTagTypeMask = (1 << kTagTypeBits) - 1; + static constexpr uint32 kTagTypeMask = (1 << kTagTypeBits) - 1; // Helper functions for encoding and decoding tags. (Inlined below and in // _inl.h) // - // This is different from MakeTag(field->number(), field->type()) in the case - // of packed repeated fields. - static uint32 MakeTag(int field_number, WireType type); + // This is different from MakeTag(field->number(), field->type()) in the + // case of packed repeated fields. + constexpr static uint32 MakeTag(int field_number, WireType type); static WireType GetTagWireType(uint32 tag); static int GetTagFieldNumber(uint32 tag); @@ -177,9 +175,9 @@ class LIBPROTOBUF_EXPORT WireFormatLite { WireFormatLite::FieldType type); // Skips a field value with the given tag. The input should start - // positioned immediately after the tag. Skipped values are simply discarded, - // not recorded anywhere. See WireFormat::SkipField() for a version that - // records to an UnknownFieldSet. + // positioned immediately after the tag. Skipped values are simply + // discarded, not recorded anywhere. See WireFormat::SkipField() for a + // version that records to an UnknownFieldSet. static bool SkipField(io::CodedInputStream* input, uint32 tag); // Skips a field value with the given tag. The input should start @@ -198,14 +196,12 @@ class LIBPROTOBUF_EXPORT WireFormatLite { static bool SkipMessage(io::CodedInputStream* input, io::CodedOutputStream* output); -// This macro does the same thing as WireFormatLite::MakeTag(), but the -// result is usable as a compile-time constant, which makes it usable -// as a switch case or a template input. WireFormatLite::MakeTag() is more -// type-safe, though, so prefer it if possible. -#define GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(FIELD_NUMBER, TYPE) \ - static_cast( \ - (static_cast(FIELD_NUMBER) << ::google::protobuf::internal::WireFormatLite::kTagTypeBits) \ - | (TYPE)) + // This macro does the same thing as WireFormatLite::MakeTag(), but the + // result is usable as a compile-time constant, which makes it usable + // as a switch case or a template input. WireFormatLite::MakeTag() is more + // type-safe, though, so prefer it if possible. +#define GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(FIELD_NUMBER, TYPE) \ + static_cast((static_cast(FIELD_NUMBER) << 3) | (TYPE)) // These are the tags for the old MessageSet format, which was defined as: // message MessageSet { @@ -214,21 +210,17 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // required string message = 3; // } // } - static const int kMessageSetItemNumber = 1; - static const int kMessageSetTypeIdNumber = 2; - static const int kMessageSetMessageNumber = 3; - static const int kMessageSetItemStartTag = - GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(kMessageSetItemNumber, - WireFormatLite::WIRETYPE_START_GROUP); - static const int kMessageSetItemEndTag = - GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(kMessageSetItemNumber, - WireFormatLite::WIRETYPE_END_GROUP); - static const int kMessageSetTypeIdTag = - GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(kMessageSetTypeIdNumber, - WireFormatLite::WIRETYPE_VARINT); - static const int kMessageSetMessageTag = - GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(kMessageSetMessageNumber, - WireFormatLite::WIRETYPE_LENGTH_DELIMITED); + static constexpr int kMessageSetItemNumber = 1; + static constexpr int kMessageSetTypeIdNumber = 2; + static constexpr int kMessageSetMessageNumber = 3; + static const int kMessageSetItemStartTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG( + kMessageSetItemNumber, WireFormatLite::WIRETYPE_START_GROUP); + static const int kMessageSetItemEndTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG( + kMessageSetItemNumber, WireFormatLite::WIRETYPE_END_GROUP); + static const int kMessageSetTypeIdTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG( + kMessageSetTypeIdNumber, WireFormatLite::WIRETYPE_VARINT); + static const int kMessageSetMessageTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG( + kMessageSetMessageNumber, WireFormatLite::WIRETYPE_LENGTH_DELIMITED); // Byte size of all tags of a MessageSet::Item combined. static const size_t kMessageSetItemTagsSize; @@ -248,24 +240,12 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // the purpose of varint. So, for the "sint32" and "sint64" field types, // we ZigZag-encode the values. static uint32 ZigZagEncode32(int32 n); - static int32 ZigZagDecode32(uint32 n); + static int32 ZigZagDecode32(uint32 n); static uint64 ZigZagEncode64(int64 n); - static int64 ZigZagDecode64(uint64 n); + static int64 ZigZagDecode64(uint64 n); // ================================================================= - // Methods for reading/writing individual field. The implementations - // of these methods are defined in wire_format_lite_inl.h; you must #include - // that file to use these. - -#ifdef NDEBUG -#define INL GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -#else -// Avoid excessive inlining in non-optimized builds. Without other optimizations -// the inlining is not going to provide benefits anyway and the huge resulting -// functions, especially in the proto-generated serialization functions, produce -// stack frames so large that many tests run into stack overflows (b/32192897). -#define INL -#endif + // Methods for reading/writing individual field. // Read fields, not including tags. The assumption is that you already // read the tag to determine what field to read. @@ -274,15 +254,16 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // the represented type and the FieldType. These are specialized with the // appropriate definition for each declared type. template - INL static bool ReadPrimitive(io::CodedInputStream* input, CType* value); + PROTOBUF_ALWAYS_INLINE static bool ReadPrimitive(io::CodedInputStream* input, + CType* value); // Reads repeated primitive values, with optimizations for repeats. // tag_size and tag should both be compile-time constants provided by the // protocol compiler. template - INL static bool ReadRepeatedPrimitive(int tag_size, uint32 tag, - io::CodedInputStream* input, - RepeatedField* value); + PROTOBUF_ALWAYS_INLINE static bool ReadRepeatedPrimitive( + int tag_size, uint32 tag, io::CodedInputStream* input, + RepeatedField* value); // Identical to ReadRepeatedPrimitive, except will not inline the // implementation. @@ -296,15 +277,16 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // // This is only implemented for the types with fixed wire size, e.g. // float, double, and the (s)fixed* types. - template INL - static const uint8* ReadPrimitiveFromArray(const uint8* buffer, CType* value); + template + PROTOBUF_ALWAYS_INLINE static const uint8* ReadPrimitiveFromArray( + const uint8* buffer, CType* value); // Reads a primitive packed field. // // This is only implemented for packable types. template - INL static bool ReadPackedPrimitive(io::CodedInputStream* input, - RepeatedField* value); + PROTOBUF_ALWAYS_INLINE static bool ReadPackedPrimitive( + io::CodedInputStream* input, RepeatedField* value); // Identical to ReadPackedPrimitive, except will not inline the // implementation. @@ -324,17 +306,19 @@ class LIBPROTOBUF_EXPORT WireFormatLite { io::CodedInputStream* input, int field_number, bool (*is_valid)(int), io::CodedOutputStream* unknown_fields_stream, RepeatedField* values); - // Read a string. ReadString(..., string* value) requires an existing string. - static inline bool ReadString(io::CodedInputStream* input, string* value); - // ReadString(..., string** p) is internal-only, and should only be called - // from generated code. It starts by setting *p to "new string" - // if *p == &GetEmptyStringAlreadyInited(). It then invokes + // Read a string. ReadString(..., std::string* value) requires an + // existing std::string. + static inline bool ReadString(io::CodedInputStream* input, + std::string* value); + // ReadString(..., std::string** p) is internal-only, and should only be + // called from generated code. It starts by setting *p to "new std::string" if + // *p == &GetEmptyStringAlreadyInited(). It then invokes // ReadString(io::CodedInputStream* input, *p). This is useful for reducing // code size. - static inline bool ReadString(io::CodedInputStream* input, string** p); + static inline bool ReadString(io::CodedInputStream* input, std::string** p); // Analogous to ReadString(). - static bool ReadBytes(io::CodedInputStream* input, string* value); - static bool ReadBytes(io::CodedInputStream* input, string** p); + static bool ReadBytes(io::CodedInputStream* input, std::string* value); + static bool ReadBytes(io::CodedInputStream* input, std::string** p); enum Operation { PARSE = 0, @@ -342,8 +326,7 @@ class LIBPROTOBUF_EXPORT WireFormatLite { }; // Returns true if the data is valid UTF-8. - static bool VerifyUtf8String(const char* data, int size, - Operation op, + static bool VerifyUtf8String(const char* data, int size, Operation op, const char* field_name); template @@ -354,15 +337,7 @@ class LIBPROTOBUF_EXPORT WireFormatLite { static inline bool ReadMessage(io::CodedInputStream* input, MessageType* value); - // Do not use. template - static inline bool ReadGroupNoVirtual(int field_number, - io::CodedInputStream* input, - MessageType* value) { - return ReadGroup(field_number, input, value); - } - - template static inline bool ReadMessageNoVirtual(io::CodedInputStream* input, MessageType* value) { return ReadMessage(input, value); @@ -371,28 +346,38 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // Write a tag. The Write*() functions typically include the tag, so // normally there's no need to call this unless using the Write*NoTag() // variants. - INL static void WriteTag(int field_number, WireType type, - io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteTag(int field_number, WireType type, + io::CodedOutputStream* output); // Write fields, without tags. - INL static void WriteInt32NoTag(int32 value, io::CodedOutputStream* output); - INL static void WriteInt64NoTag(int64 value, io::CodedOutputStream* output); - INL static void WriteUInt32NoTag(uint32 value, io::CodedOutputStream* output); - INL static void WriteUInt64NoTag(uint64 value, io::CodedOutputStream* output); - INL static void WriteSInt32NoTag(int32 value, io::CodedOutputStream* output); - INL static void WriteSInt64NoTag(int64 value, io::CodedOutputStream* output); - INL static void WriteFixed32NoTag(uint32 value, - io::CodedOutputStream* output); - INL static void WriteFixed64NoTag(uint64 value, - io::CodedOutputStream* output); - INL static void WriteSFixed32NoTag(int32 value, - io::CodedOutputStream* output); - INL static void WriteSFixed64NoTag(int64 value, - io::CodedOutputStream* output); - INL static void WriteFloatNoTag(float value, io::CodedOutputStream* output); - INL static void WriteDoubleNoTag(double value, io::CodedOutputStream* output); - INL static void WriteBoolNoTag(bool value, io::CodedOutputStream* output); - INL static void WriteEnumNoTag(int value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteInt32NoTag( + int32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteInt64NoTag( + int64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteUInt32NoTag( + uint32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteUInt64NoTag( + uint64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSInt32NoTag( + int32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSInt64NoTag( + int64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteFixed32NoTag( + uint32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteFixed64NoTag( + uint64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSFixed32NoTag( + int32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSFixed64NoTag( + int64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteFloatNoTag( + float value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteDoubleNoTag( + double value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteBoolNoTag( + bool value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteEnumNoTag( + int value, io::CodedOutputStream* output); // Write array of primitive fields, without tags static void WriteFloatArray(const float* a, int n, @@ -440,13 +425,14 @@ class LIBPROTOBUF_EXPORT WireFormatLite { static void WriteEnum(int field_number, int value, io::CodedOutputStream* output); - static void WriteString(int field_number, const string& value, + static void WriteString(int field_number, const std::string& value, io::CodedOutputStream* output); - static void WriteBytes(int field_number, const string& value, + static void WriteBytes(int field_number, const std::string& value, io::CodedOutputStream* output); - static void WriteStringMaybeAliased(int field_number, const string& value, + static void WriteStringMaybeAliased(int field_number, + const std::string& value, io::CodedOutputStream* output); - static void WriteBytesMaybeAliased(int field_number, const string& value, + static void WriteBytesMaybeAliased(int field_number, const std::string& value, io::CodedOutputStream* output); static void WriteGroup(int field_number, const MessageLite& value, @@ -474,234 +460,253 @@ class LIBPROTOBUF_EXPORT WireFormatLite { io::CodedOutputStream* output); // Like above, but use only *ToArray methods of CodedOutputStream. - INL static uint8* WriteTagToArray(int field_number, WireType type, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteTagToArray(int field_number, + WireType type, + uint8* target); // Write fields, without tags. - INL static uint8* WriteInt32NoTagToArray(int32 value, uint8* target); - INL static uint8* WriteInt64NoTagToArray(int64 value, uint8* target); - INL static uint8* WriteUInt32NoTagToArray(uint32 value, uint8* target); - INL static uint8* WriteUInt64NoTagToArray(uint64 value, uint8* target); - INL static uint8* WriteSInt32NoTagToArray(int32 value, uint8* target); - INL static uint8* WriteSInt64NoTagToArray(int64 value, uint8* target); - INL static uint8* WriteFixed32NoTagToArray(uint32 value, uint8* target); - INL static uint8* WriteFixed64NoTagToArray(uint64 value, uint8* target); - INL static uint8* WriteSFixed32NoTagToArray(int32 value, uint8* target); - INL static uint8* WriteSFixed64NoTagToArray(int64 value, uint8* target); - INL static uint8* WriteFloatNoTagToArray(float value, uint8* target); - INL static uint8* WriteDoubleNoTagToArray(double value, uint8* target); - INL static uint8* WriteBoolNoTagToArray(bool value, uint8* target); - INL static uint8* WriteEnumNoTagToArray(int value, uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32NoTagToArray(int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64NoTagToArray(int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32NoTagToArray(uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64NoTagToArray(uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32NoTagToArray(int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64NoTagToArray(int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32NoTagToArray(uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64NoTagToArray(uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32NoTagToArray(int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64NoTagToArray(int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatNoTagToArray(float value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleNoTagToArray(double value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolNoTagToArray(bool value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumNoTagToArray(int value, + uint8* target); // Write fields, without tags. These require that value.size() > 0. - template - INL static uint8* WritePrimitiveNoTagToArray( - const RepeatedField& value, - uint8* (*Writer)(T, uint8*), uint8* target); - template - INL static uint8* WriteFixedNoTagToArray( - const RepeatedField& value, - uint8* (*Writer)(T, uint8*), uint8* target); - - INL static uint8* WriteInt32NoTagToArray( - const RepeatedField< int32>& value, uint8* output); - INL static uint8* WriteInt64NoTagToArray( - const RepeatedField< int64>& value, uint8* output); - INL static uint8* WriteUInt32NoTagToArray( + template + PROTOBUF_ALWAYS_INLINE static uint8* WritePrimitiveNoTagToArray( + const RepeatedField& value, uint8* (*Writer)(T, uint8*), + uint8* target); + template + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixedNoTagToArray( + const RepeatedField& value, uint8* (*Writer)(T, uint8*), + uint8* target); + + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32NoTagToArray( const RepeatedField& value, uint8* output); - INL static uint8* WriteUInt64NoTagToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64NoTagToArray( const RepeatedField& value, uint8* output); - INL static uint8* WriteSInt32NoTagToArray( - const RepeatedField< int32>& value, uint8* output); - INL static uint8* WriteSInt64NoTagToArray( - const RepeatedField< int64>& value, uint8* output); - INL static uint8* WriteFixed32NoTagToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32NoTagToArray( const RepeatedField& value, uint8* output); - INL static uint8* WriteFixed64NoTagToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64NoTagToArray( const RepeatedField& value, uint8* output); - INL static uint8* WriteSFixed32NoTagToArray( - const RepeatedField< int32>& value, uint8* output); - INL static uint8* WriteSFixed64NoTagToArray( - const RepeatedField< int64>& value, uint8* output); - INL static uint8* WriteFloatNoTagToArray( - const RepeatedField< float>& value, uint8* output); - INL static uint8* WriteDoubleNoTagToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatNoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleNoTagToArray( const RepeatedField& value, uint8* output); - INL static uint8* WriteBoolNoTagToArray( - const RepeatedField< bool>& value, uint8* output); - INL static uint8* WriteEnumNoTagToArray( - const RepeatedField< int>& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolNoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumNoTagToArray( + const RepeatedField& value, uint8* output); // Write fields, including tags. - INL static uint8* WriteInt32ToArray(int field_number, int32 value, - uint8* target); - INL static uint8* WriteInt64ToArray(int field_number, int64 value, - uint8* target); - INL static uint8* WriteUInt32ToArray(int field_number, uint32 value, - uint8* target); - INL static uint8* WriteUInt64ToArray(int field_number, uint64 value, - uint8* target); - INL static uint8* WriteSInt32ToArray(int field_number, int32 value, - uint8* target); - INL static uint8* WriteSInt64ToArray(int field_number, int64 value, - uint8* target); - INL static uint8* WriteFixed32ToArray(int field_number, uint32 value, - uint8* target); - INL static uint8* WriteFixed64ToArray(int field_number, uint64 value, - uint8* target); - INL static uint8* WriteSFixed32ToArray(int field_number, int32 value, - uint8* target); - INL static uint8* WriteSFixed64ToArray(int field_number, int64 value, - uint8* target); - INL static uint8* WriteFloatToArray(int field_number, float value, - uint8* target); - INL static uint8* WriteDoubleToArray(int field_number, double value, - uint8* target); - INL static uint8* WriteBoolToArray(int field_number, bool value, - uint8* target); - INL static uint8* WriteEnumToArray(int field_number, int value, - uint8* target); - - template - INL static uint8* WritePrimitiveToArray( - int field_number, - const RepeatedField& value, + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32ToArray(int field_number, + int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64ToArray(int field_number, + int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32ToArray(int field_number, + uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64ToArray(int field_number, + uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32ToArray(int field_number, + int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64ToArray(int field_number, + int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32ToArray(int field_number, + uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64ToArray(int field_number, + uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32ToArray(int field_number, + int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64ToArray(int field_number, + int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatToArray(int field_number, + float value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleToArray(int field_number, + double value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolToArray(int field_number, + bool value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumToArray(int field_number, + int value, + uint8* target); + + template + PROTOBUF_ALWAYS_INLINE static uint8* WritePrimitiveToArray( + int field_number, const RepeatedField& value, uint8* (*Writer)(int, T, uint8*), uint8* target); - INL static uint8* WriteInt32ToArray( - int field_number, const RepeatedField< int32>& value, uint8* output); - INL static uint8* WriteInt64ToArray( - int field_number, const RepeatedField< int64>& value, uint8* output); - INL static uint8* WriteUInt32ToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32ToArray( int field_number, const RepeatedField& value, uint8* output); - INL static uint8* WriteUInt64ToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64ToArray( int field_number, const RepeatedField& value, uint8* output); - INL static uint8* WriteSInt32ToArray( - int field_number, const RepeatedField< int32>& value, uint8* output); - INL static uint8* WriteSInt64ToArray( - int field_number, const RepeatedField< int64>& value, uint8* output); - INL static uint8* WriteFixed32ToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32ToArray( int field_number, const RepeatedField& value, uint8* output); - INL static uint8* WriteFixed64ToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64ToArray( int field_number, const RepeatedField& value, uint8* output); - INL static uint8* WriteSFixed32ToArray( - int field_number, const RepeatedField< int32>& value, uint8* output); - INL static uint8* WriteSFixed64ToArray( - int field_number, const RepeatedField< int64>& value, uint8* output); - INL static uint8* WriteFloatToArray( - int field_number, const RepeatedField< float>& value, uint8* output); - INL static uint8* WriteDoubleToArray( + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleToArray( int field_number, const RepeatedField& value, uint8* output); - INL static uint8* WriteBoolToArray( - int field_number, const RepeatedField< bool>& value, uint8* output); - INL static uint8* WriteEnumToArray( - int field_number, const RepeatedField< int>& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumToArray( + int field_number, const RepeatedField& value, uint8* output); - INL static uint8* WriteStringToArray(int field_number, const string& value, - uint8* target); - INL static uint8* WriteBytesToArray(int field_number, const string& value, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteStringToArray( + int field_number, const std::string& value, uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBytesToArray( + int field_number, const std::string& value, uint8* target); // Whether to serialize deterministically (e.g., map keys are // sorted) is a property of a CodedOutputStream, and in the process // of serialization, the "ToArray" variants may be invoked. But they don't // have a CodedOutputStream available, so they get an additional parameter // telling them whether to serialize deterministically. - template - INL static uint8* InternalWriteGroupToArray(int field_number, - const MessageType& value, - bool deterministic, - uint8* target); - template - INL static uint8* InternalWriteMessageToArray(int field_number, - const MessageType& value, - bool deterministic, - uint8* target); + template + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteGroup( + int field_number, const MessageType& value, uint8* target, + io::EpsCopyOutputStream* stream); + template + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteMessage( + int field_number, const MessageType& value, uint8* target, + io::EpsCopyOutputStream* stream); // Like above, but de-virtualize the call to SerializeWithCachedSizes(). The // pointer must point at an instance of MessageType, *not* a subclass (or // the subclass must not override SerializeWithCachedSizes()). template - INL static uint8* InternalWriteGroupNoVirtualToArray(int field_number, - const MessageType& value, - bool deterministic, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteGroupNoVirtualToArray( + int field_number, const MessageType& value, uint8* target); template - INL static uint8* InternalWriteMessageNoVirtualToArray( - int field_number, const MessageType& value, bool deterministic, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteMessageNoVirtualToArray( + int field_number, const MessageType& value, uint8* target); // For backward-compatibility, the last four methods also have versions // that are non-deterministic always. - INL static uint8* WriteGroupToArray(int field_number, - const MessageLite& value, uint8* target) { - return InternalWriteGroupToArray(field_number, value, false, target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteGroupToArray( + int field_number, const MessageLite& value, uint8* target) { + io::EpsCopyOutputStream stream( + target, + value.GetCachedSize() + + static_cast(2 * io::CodedOutputStream::VarintSize32( + static_cast(field_number) << 3)), + io::CodedOutputStream::IsDefaultSerializationDeterministic()); + return InternalWriteGroup(field_number, value, target, &stream); } - INL static uint8* WriteMessageToArray(int field_number, - const MessageLite& value, - uint8* target) { - return InternalWriteMessageToArray(field_number, value, false, target); - } - template - INL static uint8* WriteGroupNoVirtualToArray(int field_number, - const MessageType& value, - uint8* target) { - return InternalWriteGroupNoVirtualToArray(field_number, value, false, - target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteMessageToArray( + int field_number, const MessageLite& value, uint8* target) { + int size = value.GetCachedSize(); + io::EpsCopyOutputStream stream( + target, + size + static_cast(io::CodedOutputStream::VarintSize32( + static_cast(field_number) << 3) + + io::CodedOutputStream::VarintSize32(size)), + io::CodedOutputStream::IsDefaultSerializationDeterministic()); + return InternalWriteMessage(field_number, value, target, &stream); } - template - INL static uint8* WriteMessageNoVirtualToArray(int field_number, - const MessageType& value, - uint8* target) { - return InternalWriteMessageNoVirtualToArray(field_number, value, false, - target); - } - -#undef INL // Compute the byte size of a field. The XxSize() functions do NOT include // the tag, so you must also call TagSize(). (This is because, for repeated // fields, you should only call TagSize() once and multiply it by the element // count, but you may have to call XxSize() for each individual element.) - static inline size_t Int32Size ( int32 value); - static inline size_t Int64Size ( int64 value); - static inline size_t UInt32Size (uint32 value); - static inline size_t UInt64Size (uint64 value); - static inline size_t SInt32Size ( int32 value); - static inline size_t SInt64Size ( int64 value); - static inline size_t EnumSize ( int value); - - static size_t Int32Size (const RepeatedField< int32>& value); - static size_t Int64Size (const RepeatedField< int64>& value); + static inline size_t Int32Size(int32 value); + static inline size_t Int64Size(int64 value); + static inline size_t UInt32Size(uint32 value); + static inline size_t UInt64Size(uint64 value); + static inline size_t SInt32Size(int32 value); + static inline size_t SInt64Size(int64 value); + static inline size_t EnumSize(int value); + + static size_t Int32Size(const RepeatedField& value); + static size_t Int64Size(const RepeatedField& value); static size_t UInt32Size(const RepeatedField& value); static size_t UInt64Size(const RepeatedField& value); - static size_t SInt32Size(const RepeatedField< int32>& value); - static size_t SInt64Size(const RepeatedField< int64>& value); - static size_t EnumSize (const RepeatedField< int>& value); + static size_t SInt32Size(const RepeatedField& value); + static size_t SInt64Size(const RepeatedField& value); + static size_t EnumSize(const RepeatedField& value); // These types always have the same size. - static const size_t kFixed32Size = 4; - static const size_t kFixed64Size = 8; - static const size_t kSFixed32Size = 4; - static const size_t kSFixed64Size = 8; - static const size_t kFloatSize = 4; - static const size_t kDoubleSize = 8; - static const size_t kBoolSize = 1; - - static inline size_t StringSize(const string& value); - static inline size_t BytesSize (const string& value); - - template - static inline size_t GroupSize (const MessageType& value); - template + static constexpr size_t kFixed32Size = 4; + static constexpr size_t kFixed64Size = 8; + static constexpr size_t kSFixed32Size = 4; + static constexpr size_t kSFixed64Size = 8; + static constexpr size_t kFloatSize = 4; + static constexpr size_t kDoubleSize = 8; + static constexpr size_t kBoolSize = 1; + + static inline size_t StringSize(const std::string& value); + static inline size_t BytesSize(const std::string& value); + + template + static inline size_t GroupSize(const MessageType& value); + template static inline size_t MessageSize(const MessageType& value); // Like above, but de-virtualize the call to ByteSize(). The // pointer must point at an instance of MessageType, *not* a subclass (or // the subclass must not override ByteSize()). - template - static inline size_t GroupSizeNoVirtual (const MessageType& value); - template + template + static inline size_t GroupSizeNoVirtual(const MessageType& value); + template static inline size_t MessageSizeNoVirtual(const MessageType& value); // Given the length of data, calculate the byte size of the data on the @@ -713,21 +718,19 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // optimizations for primitive types that have fixed size on the wire, and // can be read using potentially faster paths. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static bool ReadRepeatedFixedSizePrimitive( - int tag_size, - uint32 tag, - google::protobuf::io::CodedInputStream* input, + PROTOBUF_ALWAYS_INLINE static bool ReadRepeatedFixedSizePrimitive( + int tag_size, uint32 tag, io::CodedInputStream* input, RepeatedField* value); // Like ReadRepeatedFixedSizePrimitive but for packed primitive fields. template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static bool ReadPackedFixedSizePrimitive( - google::protobuf::io::CodedInputStream* input, RepeatedField* value); + PROTOBUF_ALWAYS_INLINE static bool ReadPackedFixedSizePrimitive( + io::CodedInputStream* input, RepeatedField* value); static const CppType kFieldTypeToCppTypeMap[]; static const WireFormatLite::WireType kWireTypeForFieldType[]; + static void WriteSubMessageMaybeToArray(int size, const MessageLite& value, + io::CodedOutputStream* output); GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(WireFormatLite); }; @@ -736,7 +739,7 @@ class LIBPROTOBUF_EXPORT WireFormatLite { // discards them. WireFormat defines a subclass which writes to an // UnknownFieldSet. This class is used by ExtensionSet::ParseField(), since // ExtensionSet is part of the lite library but UnknownFieldSet is not. -class LIBPROTOBUF_EXPORT FieldSkipper { +class PROTOBUF_EXPORT FieldSkipper { public: FieldSkipper() {} virtual ~FieldSkipper() {} @@ -756,30 +759,30 @@ class LIBPROTOBUF_EXPORT FieldSkipper { // Subclass of FieldSkipper which saves skipped fields to a CodedOutputStream. -class LIBPROTOBUF_EXPORT CodedOutputStreamFieldSkipper : public FieldSkipper { +class PROTOBUF_EXPORT CodedOutputStreamFieldSkipper : public FieldSkipper { public: explicit CodedOutputStreamFieldSkipper(io::CodedOutputStream* unknown_fields) : unknown_fields_(unknown_fields) {} - virtual ~CodedOutputStreamFieldSkipper() {} + ~CodedOutputStreamFieldSkipper() override {} // implements FieldSkipper ----------------------------------------- - virtual bool SkipField(io::CodedInputStream* input, uint32 tag); - virtual bool SkipMessage(io::CodedInputStream* input); - virtual void SkipUnknownEnum(int field_number, int value); + bool SkipField(io::CodedInputStream* input, uint32 tag) override; + bool SkipMessage(io::CodedInputStream* input) override; + void SkipUnknownEnum(int field_number, int value) override; protected: io::CodedOutputStream* unknown_fields_; }; - // inline methods ==================================================== -inline WireFormatLite::CppType -WireFormatLite::FieldTypeToCppType(FieldType type) { +inline WireFormatLite::CppType WireFormatLite::FieldTypeToCppType( + FieldType type) { return kFieldTypeToCppTypeMap[type]; } -inline uint32 WireFormatLite::MakeTag(int field_number, WireType type) { +constexpr inline uint32 WireFormatLite::MakeTag(int field_number, + WireType type) { return GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(field_number, type); } @@ -794,7 +797,7 @@ inline int WireFormatLite::GetTagFieldNumber(uint32 tag) { inline size_t WireFormatLite::TagSize(int field_number, WireFormatLite::FieldType type) { size_t result = io::CodedOutputStream::VarintSize32( - static_cast(field_number << kTagTypeBits)); + static_cast(field_number << kTagTypeBits)); if (type == TYPE_GROUP) { // Groups have both a start and an end tag. return result * 2; @@ -804,27 +807,19 @@ inline size_t WireFormatLite::TagSize(int field_number, } inline uint32 WireFormatLite::EncodeFloat(float value) { - union {float f; uint32 i;}; - f = value; - return i; + return bit_cast(value); } inline float WireFormatLite::DecodeFloat(uint32 value) { - union {float f; uint32 i;}; - i = value; - return f; + return bit_cast(value); } inline uint64 WireFormatLite::EncodeDouble(double value) { - union {double f; uint64 i;}; - f = value; - return i; + return bit_cast(value); } inline double WireFormatLite::DecodeDouble(uint64 value) { - union {double f; uint64 i;}; - i = value; - return f; + return bit_cast(value); } // ZigZag Transform: Encodes signed integers so that they can be @@ -877,17 +872,995 @@ inline int64 WireFormatLite::ZigZagDecode64(uint64 n) { // call ReadBytes(). inline bool WireFormatLite::ReadString(io::CodedInputStream* input, - string* value) { + std::string* value) { return ReadBytes(input, value); } inline bool WireFormatLite::ReadString(io::CodedInputStream* input, - string** p) { + std::string** p) { return ReadBytes(input, p); } +inline uint8* InternalSerializeUnknownMessageSetItemsToArray( + const std::string& unknown_fields, uint8* target, + io::EpsCopyOutputStream* stream) { + return stream->WriteRaw(unknown_fields.data(), + static_cast(unknown_fields.size()), target); +} + +inline size_t ComputeUnknownMessageSetItemsSize( + const std::string& unknown_fields) { + return unknown_fields.size(); +} + +// Implementation details of ReadPrimitive. + +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, int32* value) { + uint32 temp; + if (!input->ReadVarint32(&temp)) return false; + *value = static_cast(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, int64* value) { + uint64 temp; + if (!input->ReadVarint64(&temp)) return false; + *value = static_cast(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, uint32* value) { + return input->ReadVarint32(value); +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, uint64* value) { + return input->ReadVarint64(value); +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, int32* value) { + uint32 temp; + if (!input->ReadVarint32(&temp)) return false; + *value = ZigZagDecode32(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, int64* value) { + uint64 temp; + if (!input->ReadVarint64(&temp)) return false; + *value = ZigZagDecode64(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, uint32* value) { + return input->ReadLittleEndian32(value); +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, uint64* value) { + return input->ReadLittleEndian64(value); +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, int32* value) { + uint32 temp; + if (!input->ReadLittleEndian32(&temp)) return false; + *value = static_cast(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, int64* value) { + uint64 temp; + if (!input->ReadLittleEndian64(&temp)) return false; + *value = static_cast(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, float* value) { + uint32 temp; + if (!input->ReadLittleEndian32(&temp)) return false; + *value = DecodeFloat(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, double* value) { + uint64 temp; + if (!input->ReadLittleEndian64(&temp)) return false; + *value = DecodeDouble(temp); + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, bool* value) { + uint64 temp; + if (!input->ReadVarint64(&temp)) return false; + *value = temp != 0; + return true; +} +template <> +inline bool WireFormatLite::ReadPrimitive( + io::CodedInputStream* input, int* value) { + uint32 temp; + if (!input->ReadVarint32(&temp)) return false; + *value = static_cast(temp); + return true; +} + +template <> +inline const uint8* +WireFormatLite::ReadPrimitiveFromArray( + const uint8* buffer, uint32* value) { + return io::CodedInputStream::ReadLittleEndian32FromArray(buffer, value); +} +template <> +inline const uint8* +WireFormatLite::ReadPrimitiveFromArray( + const uint8* buffer, uint64* value) { + return io::CodedInputStream::ReadLittleEndian64FromArray(buffer, value); +} +template <> +inline const uint8* +WireFormatLite::ReadPrimitiveFromArray( + const uint8* buffer, int32* value) { + uint32 temp; + buffer = io::CodedInputStream::ReadLittleEndian32FromArray(buffer, &temp); + *value = static_cast(temp); + return buffer; +} +template <> +inline const uint8* +WireFormatLite::ReadPrimitiveFromArray( + const uint8* buffer, int64* value) { + uint64 temp; + buffer = io::CodedInputStream::ReadLittleEndian64FromArray(buffer, &temp); + *value = static_cast(temp); + return buffer; +} +template <> +inline const uint8* +WireFormatLite::ReadPrimitiveFromArray( + const uint8* buffer, float* value) { + uint32 temp; + buffer = io::CodedInputStream::ReadLittleEndian32FromArray(buffer, &temp); + *value = DecodeFloat(temp); + return buffer; +} +template <> +inline const uint8* +WireFormatLite::ReadPrimitiveFromArray( + const uint8* buffer, double* value) { + uint64 temp; + buffer = io::CodedInputStream::ReadLittleEndian64FromArray(buffer, &temp); + *value = DecodeDouble(temp); + return buffer; +} + +template +inline bool WireFormatLite::ReadRepeatedPrimitive( + int, // tag_size, unused. + uint32 tag, io::CodedInputStream* input, RepeatedField* values) { + CType value; + if (!ReadPrimitive(input, &value)) return false; + values->Add(value); + int elements_already_reserved = values->Capacity() - values->size(); + while (elements_already_reserved > 0 && input->ExpectTag(tag)) { + if (!ReadPrimitive(input, &value)) return false; + values->AddAlreadyReserved(value); + elements_already_reserved--; + } + return true; +} + +template +inline bool WireFormatLite::ReadRepeatedFixedSizePrimitive( + int tag_size, uint32 tag, io::CodedInputStream* input, + RepeatedField* values) { + GOOGLE_DCHECK_EQ(UInt32Size(tag), static_cast(tag_size)); + CType value; + if (!ReadPrimitive(input, &value)) return false; + values->Add(value); + + // For fixed size values, repeated values can be read more quickly by + // reading directly from a raw array. + // + // We can get a tight loop by only reading as many elements as can be + // added to the RepeatedField without having to do any resizing. Additionally, + // we only try to read as many elements as are available from the current + // buffer space. Doing so avoids having to perform boundary checks when + // reading the value: the maximum number of elements that can be read is + // known outside of the loop. + const void* void_pointer; + int size; + input->GetDirectBufferPointerInline(&void_pointer, &size); + if (size > 0) { + const uint8* buffer = reinterpret_cast(void_pointer); + // The number of bytes each type occupies on the wire. + const int per_value_size = tag_size + static_cast(sizeof(value)); + + // parentheses around (std::min) prevents macro expansion of min(...) + int elements_available = + (std::min)(values->Capacity() - values->size(), size / per_value_size); + int num_read = 0; + while (num_read < elements_available && + (buffer = io::CodedInputStream::ExpectTagFromArray(buffer, tag)) != + NULL) { + buffer = ReadPrimitiveFromArray(buffer, &value); + values->AddAlreadyReserved(value); + ++num_read; + } + const int read_bytes = num_read * per_value_size; + if (read_bytes > 0) { + input->Skip(read_bytes); + } + } + return true; +} + +// Specializations of ReadRepeatedPrimitive for the fixed size types, which use +// the optimized code path. +#define READ_REPEATED_FIXED_SIZE_PRIMITIVE(CPPTYPE, DECLARED_TYPE) \ + template <> \ + inline bool WireFormatLite::ReadRepeatedPrimitive< \ + CPPTYPE, WireFormatLite::DECLARED_TYPE>( \ + int tag_size, uint32 tag, io::CodedInputStream* input, \ + RepeatedField* values) { \ + return ReadRepeatedFixedSizePrimitive( \ + tag_size, tag, input, values); \ + } + +READ_REPEATED_FIXED_SIZE_PRIMITIVE(uint32, TYPE_FIXED32) +READ_REPEATED_FIXED_SIZE_PRIMITIVE(uint64, TYPE_FIXED64) +READ_REPEATED_FIXED_SIZE_PRIMITIVE(int32, TYPE_SFIXED32) +READ_REPEATED_FIXED_SIZE_PRIMITIVE(int64, TYPE_SFIXED64) +READ_REPEATED_FIXED_SIZE_PRIMITIVE(float, TYPE_FLOAT) +READ_REPEATED_FIXED_SIZE_PRIMITIVE(double, TYPE_DOUBLE) + +#undef READ_REPEATED_FIXED_SIZE_PRIMITIVE + +template +bool WireFormatLite::ReadRepeatedPrimitiveNoInline( + int tag_size, uint32 tag, io::CodedInputStream* input, + RepeatedField* value) { + return ReadRepeatedPrimitive(tag_size, tag, input, + value); +} + +template +inline bool WireFormatLite::ReadPackedPrimitive(io::CodedInputStream* input, + RepeatedField* values) { + int length; + if (!input->ReadVarintSizeAsInt(&length)) return false; + io::CodedInputStream::Limit limit = input->PushLimit(length); + while (input->BytesUntilLimit() > 0) { + CType value; + if (!ReadPrimitive(input, &value)) return false; + values->Add(value); + } + input->PopLimit(limit); + return true; +} + +template +inline bool WireFormatLite::ReadPackedFixedSizePrimitive( + io::CodedInputStream* input, RepeatedField* values) { + int length; + if (!input->ReadVarintSizeAsInt(&length)) return false; + const int old_entries = values->size(); + const int new_entries = length / static_cast(sizeof(CType)); + const int new_bytes = new_entries * static_cast(sizeof(CType)); + if (new_bytes != length) return false; + // We would *like* to pre-allocate the buffer to write into (for + // speed), but *must* avoid performing a very large allocation due + // to a malicious user-supplied "length" above. So we have a fast + // path that pre-allocates when the "length" is less than a bound. + // We determine the bound by calling BytesUntilTotalBytesLimit() and + // BytesUntilLimit(). These return -1 to mean "no limit set". + // There are four cases: + // TotalBytesLimit Limit + // -1 -1 Use slow path. + // -1 >= 0 Use fast path if length <= Limit. + // >= 0 -1 Use slow path. + // >= 0 >= 0 Use fast path if length <= min(both limits). + int64 bytes_limit = input->BytesUntilTotalBytesLimit(); + if (bytes_limit == -1) { + bytes_limit = input->BytesUntilLimit(); + } else { + // parentheses around (std::min) prevents macro expansion of min(...) + bytes_limit = + (std::min)(bytes_limit, static_cast(input->BytesUntilLimit())); + } + if (bytes_limit >= new_bytes) { + // Fast-path that pre-allocates *values to the final size. +#if defined(PROTOBUF_LITTLE_ENDIAN) + values->Resize(old_entries + new_entries, 0); + // values->mutable_data() may change after Resize(), so do this after: + void* dest = reinterpret_cast(values->mutable_data() + old_entries); + if (!input->ReadRaw(dest, new_bytes)) { + values->Truncate(old_entries); + return false; + } +#else + values->Reserve(old_entries + new_entries); + CType value; + for (int i = 0; i < new_entries; ++i) { + if (!ReadPrimitive(input, &value)) return false; + values->AddAlreadyReserved(value); + } +#endif + } else { + // This is the slow-path case where "length" may be too large to + // safely allocate. We read as much as we can into *values + // without pre-allocating "length" bytes. + CType value; + for (int i = 0; i < new_entries; ++i) { + if (!ReadPrimitive(input, &value)) return false; + values->Add(value); + } + } + return true; +} + +// Specializations of ReadPackedPrimitive for the fixed size types, which use +// an optimized code path. +#define READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(CPPTYPE, DECLARED_TYPE) \ + template <> \ + inline bool \ + WireFormatLite::ReadPackedPrimitive( \ + io::CodedInputStream * input, RepeatedField * values) { \ + return ReadPackedFixedSizePrimitive( \ + input, values); \ + } + +READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(uint32, TYPE_FIXED32) +READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(uint64, TYPE_FIXED64) +READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(int32, TYPE_SFIXED32) +READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(int64, TYPE_SFIXED64) +READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(float, TYPE_FLOAT) +READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(double, TYPE_DOUBLE) + +#undef READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE + +template +bool WireFormatLite::ReadPackedPrimitiveNoInline(io::CodedInputStream* input, + RepeatedField* values) { + return ReadPackedPrimitive(input, values); +} + + +template +inline bool WireFormatLite::ReadGroup(int field_number, + io::CodedInputStream* input, + MessageType* value) { + if (!input->IncrementRecursionDepth()) return false; + if (!value->MergePartialFromCodedStream(input)) return false; + input->UnsafeDecrementRecursionDepth(); + // Make sure the last thing read was an end tag for this group. + if (!input->LastTagWas(MakeTag(field_number, WIRETYPE_END_GROUP))) { + return false; + } + return true; +} +template +inline bool WireFormatLite::ReadMessage(io::CodedInputStream* input, + MessageType* value) { + int length; + if (!input->ReadVarintSizeAsInt(&length)) return false; + std::pair p = + input->IncrementRecursionDepthAndPushLimit(length); + if (p.second < 0 || !value->MergePartialFromCodedStream(input)) return false; + // Make sure that parsing stopped when the limit was hit, not at an endgroup + // tag. + return input->DecrementRecursionDepthAndPopLimit(p.first); +} + +// =================================================================== + +inline void WireFormatLite::WriteTag(int field_number, WireType type, + io::CodedOutputStream* output) { + output->WriteTag(MakeTag(field_number, type)); +} + +inline void WireFormatLite::WriteInt32NoTag(int32 value, + io::CodedOutputStream* output) { + output->WriteVarint32SignExtended(value); +} +inline void WireFormatLite::WriteInt64NoTag(int64 value, + io::CodedOutputStream* output) { + output->WriteVarint64(static_cast(value)); +} +inline void WireFormatLite::WriteUInt32NoTag(uint32 value, + io::CodedOutputStream* output) { + output->WriteVarint32(value); +} +inline void WireFormatLite::WriteUInt64NoTag(uint64 value, + io::CodedOutputStream* output) { + output->WriteVarint64(value); +} +inline void WireFormatLite::WriteSInt32NoTag(int32 value, + io::CodedOutputStream* output) { + output->WriteVarint32(ZigZagEncode32(value)); +} +inline void WireFormatLite::WriteSInt64NoTag(int64 value, + io::CodedOutputStream* output) { + output->WriteVarint64(ZigZagEncode64(value)); +} +inline void WireFormatLite::WriteFixed32NoTag(uint32 value, + io::CodedOutputStream* output) { + output->WriteLittleEndian32(value); +} +inline void WireFormatLite::WriteFixed64NoTag(uint64 value, + io::CodedOutputStream* output) { + output->WriteLittleEndian64(value); +} +inline void WireFormatLite::WriteSFixed32NoTag(int32 value, + io::CodedOutputStream* output) { + output->WriteLittleEndian32(static_cast(value)); +} +inline void WireFormatLite::WriteSFixed64NoTag(int64 value, + io::CodedOutputStream* output) { + output->WriteLittleEndian64(static_cast(value)); +} +inline void WireFormatLite::WriteFloatNoTag(float value, + io::CodedOutputStream* output) { + output->WriteLittleEndian32(EncodeFloat(value)); +} +inline void WireFormatLite::WriteDoubleNoTag(double value, + io::CodedOutputStream* output) { + output->WriteLittleEndian64(EncodeDouble(value)); +} +inline void WireFormatLite::WriteBoolNoTag(bool value, + io::CodedOutputStream* output) { + output->WriteVarint32(value ? 1 : 0); +} +inline void WireFormatLite::WriteEnumNoTag(int value, + io::CodedOutputStream* output) { + output->WriteVarint32SignExtended(value); +} + +// See comment on ReadGroupNoVirtual to understand the need for this template +// parameter name. +template +inline void WireFormatLite::WriteGroupNoVirtual( + int field_number, const MessageType_WorkAroundCppLookupDefect& value, + io::CodedOutputStream* output) { + WriteTag(field_number, WIRETYPE_START_GROUP, output); + value.MessageType_WorkAroundCppLookupDefect::SerializeWithCachedSizes(output); + WriteTag(field_number, WIRETYPE_END_GROUP, output); +} +template +inline void WireFormatLite::WriteMessageNoVirtual( + int field_number, const MessageType_WorkAroundCppLookupDefect& value, + io::CodedOutputStream* output) { + WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); + output->WriteVarint32( + value.MessageType_WorkAroundCppLookupDefect::GetCachedSize()); + value.MessageType_WorkAroundCppLookupDefect::SerializeWithCachedSizes(output); +} + +// =================================================================== + +inline uint8* WireFormatLite::WriteTagToArray(int field_number, WireType type, + uint8* target) { + return io::CodedOutputStream::WriteTagToArray(MakeTag(field_number, type), + target); +} + +inline uint8* WireFormatLite::WriteInt32NoTagToArray(int32 value, + uint8* target) { + return io::CodedOutputStream::WriteVarint32SignExtendedToArray(value, target); +} +inline uint8* WireFormatLite::WriteInt64NoTagToArray(int64 value, + uint8* target) { + return io::CodedOutputStream::WriteVarint64ToArray(static_cast(value), + target); +} +inline uint8* WireFormatLite::WriteUInt32NoTagToArray(uint32 value, + uint8* target) { + return io::CodedOutputStream::WriteVarint32ToArray(value, target); +} +inline uint8* WireFormatLite::WriteUInt64NoTagToArray(uint64 value, + uint8* target) { + return io::CodedOutputStream::WriteVarint64ToArray(value, target); +} +inline uint8* WireFormatLite::WriteSInt32NoTagToArray(int32 value, + uint8* target) { + return io::CodedOutputStream::WriteVarint32ToArray(ZigZagEncode32(value), + target); +} +inline uint8* WireFormatLite::WriteSInt64NoTagToArray(int64 value, + uint8* target) { + return io::CodedOutputStream::WriteVarint64ToArray(ZigZagEncode64(value), + target); +} +inline uint8* WireFormatLite::WriteFixed32NoTagToArray(uint32 value, + uint8* target) { + return io::CodedOutputStream::WriteLittleEndian32ToArray(value, target); +} +inline uint8* WireFormatLite::WriteFixed64NoTagToArray(uint64 value, + uint8* target) { + return io::CodedOutputStream::WriteLittleEndian64ToArray(value, target); +} +inline uint8* WireFormatLite::WriteSFixed32NoTagToArray(int32 value, + uint8* target) { + return io::CodedOutputStream::WriteLittleEndian32ToArray( + static_cast(value), target); +} +inline uint8* WireFormatLite::WriteSFixed64NoTagToArray(int64 value, + uint8* target) { + return io::CodedOutputStream::WriteLittleEndian64ToArray( + static_cast(value), target); +} +inline uint8* WireFormatLite::WriteFloatNoTagToArray(float value, + uint8* target) { + return io::CodedOutputStream::WriteLittleEndian32ToArray(EncodeFloat(value), + target); +} +inline uint8* WireFormatLite::WriteDoubleNoTagToArray(double value, + uint8* target) { + return io::CodedOutputStream::WriteLittleEndian64ToArray(EncodeDouble(value), + target); +} +inline uint8* WireFormatLite::WriteBoolNoTagToArray(bool value, uint8* target) { + return io::CodedOutputStream::WriteVarint32ToArray(value ? 1 : 0, target); +} +inline uint8* WireFormatLite::WriteEnumNoTagToArray(int value, uint8* target) { + return io::CodedOutputStream::WriteVarint32SignExtendedToArray(value, target); +} + +template +inline uint8* WireFormatLite::WritePrimitiveNoTagToArray( + const RepeatedField& value, uint8* (*Writer)(T, uint8*), uint8* target) { + const int n = value.size(); + GOOGLE_DCHECK_GT(n, 0); + + const T* ii = value.data(); + int i = 0; + do { + target = Writer(ii[i], target); + } while (++i < n); + + return target; +} + +template +inline uint8* WireFormatLite::WriteFixedNoTagToArray( + const RepeatedField& value, uint8* (*Writer)(T, uint8*), uint8* target) { +#if defined(PROTOBUF_LITTLE_ENDIAN) + (void)Writer; + + const int n = value.size(); + GOOGLE_DCHECK_GT(n, 0); + + const T* ii = value.data(); + const int bytes = n * static_cast(sizeof(ii[0])); + memcpy(target, ii, static_cast(bytes)); + return target + bytes; +#else + return WritePrimitiveNoTagToArray(value, Writer, target); +#endif +} + +inline uint8* WireFormatLite::WriteInt32NoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteInt32NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteInt64NoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteInt64NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteUInt32NoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteUInt32NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteUInt64NoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteUInt64NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteSInt32NoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteSInt32NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteSInt64NoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteSInt64NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteFixed32NoTagToArray( + const RepeatedField& value, uint8* target) { + return WriteFixedNoTagToArray(value, WriteFixed32NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteFixed64NoTagToArray( + const RepeatedField& value, uint8* target) { + return WriteFixedNoTagToArray(value, WriteFixed64NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteSFixed32NoTagToArray( + const RepeatedField& value, uint8* target) { + return WriteFixedNoTagToArray(value, WriteSFixed32NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteSFixed64NoTagToArray( + const RepeatedField& value, uint8* target) { + return WriteFixedNoTagToArray(value, WriteSFixed64NoTagToArray, target); +} +inline uint8* WireFormatLite::WriteFloatNoTagToArray( + const RepeatedField& value, uint8* target) { + return WriteFixedNoTagToArray(value, WriteFloatNoTagToArray, target); +} +inline uint8* WireFormatLite::WriteDoubleNoTagToArray( + const RepeatedField& value, uint8* target) { + return WriteFixedNoTagToArray(value, WriteDoubleNoTagToArray, target); +} +inline uint8* WireFormatLite::WriteBoolNoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteBoolNoTagToArray, target); +} +inline uint8* WireFormatLite::WriteEnumNoTagToArray( + const RepeatedField& value, uint8* target) { + return WritePrimitiveNoTagToArray(value, WriteEnumNoTagToArray, target); +} + +inline uint8* WireFormatLite::WriteInt32ToArray(int field_number, int32 value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteInt32NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteInt64ToArray(int field_number, int64 value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteInt64NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteUInt32ToArray(int field_number, uint32 value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteUInt32NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteUInt64ToArray(int field_number, uint64 value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteUInt64NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteSInt32ToArray(int field_number, int32 value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteSInt32NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteSInt64ToArray(int field_number, int64 value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteSInt64NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteFixed32ToArray(int field_number, + uint32 value, uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_FIXED32, target); + return WriteFixed32NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteFixed64ToArray(int field_number, + uint64 value, uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_FIXED64, target); + return WriteFixed64NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteSFixed32ToArray(int field_number, + int32 value, uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_FIXED32, target); + return WriteSFixed32NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteSFixed64ToArray(int field_number, + int64 value, uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_FIXED64, target); + return WriteSFixed64NoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteFloatToArray(int field_number, float value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_FIXED32, target); + return WriteFloatNoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteDoubleToArray(int field_number, double value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_FIXED64, target); + return WriteDoubleNoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteBoolToArray(int field_number, bool value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteBoolNoTagToArray(value, target); +} +inline uint8* WireFormatLite::WriteEnumToArray(int field_number, int value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); + return WriteEnumNoTagToArray(value, target); +} + +template +inline uint8* WireFormatLite::WritePrimitiveToArray( + int field_number, const RepeatedField& value, + uint8* (*Writer)(int, T, uint8*), uint8* target) { + const int n = value.size(); + if (n == 0) { + return target; + } + + const T* ii = value.data(); + int i = 0; + do { + target = Writer(field_number, ii[i], target); + } while (++i < n); + + return target; +} + +inline uint8* WireFormatLite::WriteInt32ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteInt32ToArray, target); +} +inline uint8* WireFormatLite::WriteInt64ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteInt64ToArray, target); +} +inline uint8* WireFormatLite::WriteUInt32ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteUInt32ToArray, target); +} +inline uint8* WireFormatLite::WriteUInt64ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteUInt64ToArray, target); +} +inline uint8* WireFormatLite::WriteSInt32ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteSInt32ToArray, target); +} +inline uint8* WireFormatLite::WriteSInt64ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteSInt64ToArray, target); +} +inline uint8* WireFormatLite::WriteFixed32ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteFixed32ToArray, + target); +} +inline uint8* WireFormatLite::WriteFixed64ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteFixed64ToArray, + target); +} +inline uint8* WireFormatLite::WriteSFixed32ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteSFixed32ToArray, + target); +} +inline uint8* WireFormatLite::WriteSFixed64ToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteSFixed64ToArray, + target); +} +inline uint8* WireFormatLite::WriteFloatToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteFloatToArray, target); +} +inline uint8* WireFormatLite::WriteDoubleToArray( + int field_number, const RepeatedField& value, uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteDoubleToArray, target); +} +inline uint8* WireFormatLite::WriteBoolToArray(int field_number, + const RepeatedField& value, + uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteBoolToArray, target); +} +inline uint8* WireFormatLite::WriteEnumToArray(int field_number, + const RepeatedField& value, + uint8* target) { + return WritePrimitiveToArray(field_number, value, WriteEnumToArray, target); +} +inline uint8* WireFormatLite::WriteStringToArray(int field_number, + const std::string& value, + uint8* target) { + // String is for UTF-8 text only + // WARNING: In wire_format.cc, both strings and bytes are handled by + // WriteString() to avoid code duplication. If the implementations become + // different, you will need to update that usage. + target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); + return io::CodedOutputStream::WriteStringWithSizeToArray(value, target); +} +inline uint8* WireFormatLite::WriteBytesToArray(int field_number, + const std::string& value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); + return io::CodedOutputStream::WriteStringWithSizeToArray(value, target); +} + + +template +inline uint8* WireFormatLite::InternalWriteGroup( + int field_number, const MessageType& value, uint8* target, + io::EpsCopyOutputStream* stream) { + target = WriteTagToArray(field_number, WIRETYPE_START_GROUP, target); + target = value._InternalSerialize(target, stream); + target = stream->EnsureSpace(target); + return WriteTagToArray(field_number, WIRETYPE_END_GROUP, target); +} +template +inline uint8* WireFormatLite::InternalWriteMessage( + int field_number, const MessageType& value, uint8* target, + io::EpsCopyOutputStream* stream) { + target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); + target = io::CodedOutputStream::WriteVarint32ToArray( + static_cast(value.GetCachedSize()), target); + return value._InternalSerialize(target, stream); +} + +// See comment on ReadGroupNoVirtual to understand the need for this template +// parameter name. +template +inline uint8* WireFormatLite::InternalWriteGroupNoVirtualToArray( + int field_number, const MessageType_WorkAroundCppLookupDefect& value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_START_GROUP, target); + target = value.MessageType_WorkAroundCppLookupDefect:: + SerializeWithCachedSizesToArray(target); + return WriteTagToArray(field_number, WIRETYPE_END_GROUP, target); +} +template +inline uint8* WireFormatLite::InternalWriteMessageNoVirtualToArray( + int field_number, const MessageType_WorkAroundCppLookupDefect& value, + uint8* target) { + target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); + target = io::CodedOutputStream::WriteVarint32ToArray( + static_cast( + value.MessageType_WorkAroundCppLookupDefect::GetCachedSize()), + target); + return value + .MessageType_WorkAroundCppLookupDefect::SerializeWithCachedSizesToArray( + target); +} + +// =================================================================== + +inline size_t WireFormatLite::Int32Size(int32 value) { + return io::CodedOutputStream::VarintSize32SignExtended(value); +} +inline size_t WireFormatLite::Int64Size(int64 value) { + return io::CodedOutputStream::VarintSize64(static_cast(value)); +} +inline size_t WireFormatLite::UInt32Size(uint32 value) { + return io::CodedOutputStream::VarintSize32(value); +} +inline size_t WireFormatLite::UInt64Size(uint64 value) { + return io::CodedOutputStream::VarintSize64(value); +} +inline size_t WireFormatLite::SInt32Size(int32 value) { + return io::CodedOutputStream::VarintSize32(ZigZagEncode32(value)); +} +inline size_t WireFormatLite::SInt64Size(int64 value) { + return io::CodedOutputStream::VarintSize64(ZigZagEncode64(value)); +} +inline size_t WireFormatLite::EnumSize(int value) { + return io::CodedOutputStream::VarintSize32SignExtended(value); +} + +inline size_t WireFormatLite::StringSize(const std::string& value) { + return LengthDelimitedSize(value.size()); +} +inline size_t WireFormatLite::BytesSize(const std::string& value) { + return LengthDelimitedSize(value.size()); +} + + +template +inline size_t WireFormatLite::GroupSize(const MessageType& value) { + return value.ByteSizeLong(); +} +template +inline size_t WireFormatLite::MessageSize(const MessageType& value) { + return LengthDelimitedSize(value.ByteSizeLong()); +} + +// See comment on ReadGroupNoVirtual to understand the need for this template +// parameter name. +template +inline size_t WireFormatLite::GroupSizeNoVirtual( + const MessageType_WorkAroundCppLookupDefect& value) { + return value.MessageType_WorkAroundCppLookupDefect::ByteSizeLong(); +} +template +inline size_t WireFormatLite::MessageSizeNoVirtual( + const MessageType_WorkAroundCppLookupDefect& value) { + return LengthDelimitedSize( + value.MessageType_WorkAroundCppLookupDefect::ByteSizeLong()); +} + +inline size_t WireFormatLite::LengthDelimitedSize(size_t length) { + // The static_cast here prevents an error in certain compiler configurations + // but is not technically correct--if length is too large to fit in a uint32 + // then it will be silently truncated. We will need to fix this if we ever + // decide to start supporting serialized messages greater than 2 GiB in size. + return length + + io::CodedOutputStream::VarintSize32(static_cast(length)); +} + +template +bool ParseMessageSetItemImpl(io::CodedInputStream* input, MS ms) { + // This method parses a group which should contain two fields: + // required int32 type_id = 2; + // required data message = 3; + + uint32 last_type_id = 0; + + // If we see message data before the type_id, we'll append it to this so + // we can parse it later. + std::string message_data; + + while (true) { + const uint32 tag = input->ReadTagNoLastTag(); + if (tag == 0) return false; + + switch (tag) { + case WireFormatLite::kMessageSetTypeIdTag: { + uint32 type_id; + if (!input->ReadVarint32(&type_id)) return false; + last_type_id = type_id; + + if (!message_data.empty()) { + // We saw some message data before the type_id. Have to parse it + // now. + io::CodedInputStream sub_input( + reinterpret_cast(message_data.data()), + static_cast(message_data.size())); + sub_input.SetRecursionLimit(input->RecursionBudget()); + if (!ms.ParseField(last_type_id, &sub_input)) { + return false; + } + message_data.clear(); + } + + break; + } + + case WireFormatLite::kMessageSetMessageTag: { + if (last_type_id == 0) { + // We haven't seen a type_id yet. Append this data to message_data. + uint32 length; + if (!input->ReadVarint32(&length)) return false; + if (static_cast(length) < 0) return false; + uint32 size = static_cast( + length + io::CodedOutputStream::VarintSize32(length)); + message_data.resize(size); + auto ptr = reinterpret_cast(&message_data[0]); + ptr = io::CodedOutputStream::WriteVarint32ToArray(length, ptr); + if (!input->ReadRaw(ptr, length)) return false; + } else { + // Already saw type_id, so we can parse this directly. + if (!ms.ParseField(last_type_id, input)) { + return false; + } + } + + break; + } + + case WireFormatLite::kMessageSetItemEndTag: { + return true; + } + + default: { + if (!ms.SkipField(tag, input)) return false; + } + } + } +} + } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_H__ diff --git a/third_party/protobuf-lite/google/protobuf/wire_format_lite_inl.h b/third_party/protobuf-lite/google/protobuf/wire_format_lite_inl.h deleted file mode 100644 index 6cd2c2fb..00000000 --- a/third_party/protobuf-lite/google/protobuf/wire_format_lite_inl.h +++ /dev/null @@ -1,996 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// wink@google.com (Wink Saville) (refactored from wire_format.h) -// Based on original Protocol Buffers design by -// Sanjay Ghemawat, Jeff Dean, and others. - -#ifndef GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__ -#define GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace google { -namespace protobuf { -namespace internal { - -// Implementation details of ReadPrimitive. - -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - int32* value) { - uint32 temp; - if (!input->ReadVarint32(&temp)) return false; - *value = static_cast(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - int64* value) { - uint64 temp; - if (!input->ReadVarint64(&temp)) return false; - *value = static_cast(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - uint32* value) { - return input->ReadVarint32(value); -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - uint64* value) { - return input->ReadVarint64(value); -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - int32* value) { - uint32 temp; - if (!input->ReadVarint32(&temp)) return false; - *value = ZigZagDecode32(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - int64* value) { - uint64 temp; - if (!input->ReadVarint64(&temp)) return false; - *value = ZigZagDecode64(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - uint32* value) { - return input->ReadLittleEndian32(value); -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - uint64* value) { - return input->ReadLittleEndian64(value); -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - int32* value) { - uint32 temp; - if (!input->ReadLittleEndian32(&temp)) return false; - *value = static_cast(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - int64* value) { - uint64 temp; - if (!input->ReadLittleEndian64(&temp)) return false; - *value = static_cast(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - float* value) { - uint32 temp; - if (!input->ReadLittleEndian32(&temp)) return false; - *value = DecodeFloat(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - double* value) { - uint64 temp; - if (!input->ReadLittleEndian64(&temp)) return false; - *value = DecodeDouble(temp); - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - bool* value) { - uint64 temp; - if (!input->ReadVarint64(&temp)) return false; - *value = temp != 0; - return true; -} -template <> -inline bool WireFormatLite::ReadPrimitive( - io::CodedInputStream* input, - int* value) { - uint32 temp; - if (!input->ReadVarint32(&temp)) return false; - *value = static_cast(temp); - return true; -} - -template <> -inline const uint8* WireFormatLite::ReadPrimitiveFromArray< - uint32, WireFormatLite::TYPE_FIXED32>( - const uint8* buffer, - uint32* value) { - return io::CodedInputStream::ReadLittleEndian32FromArray(buffer, value); -} -template <> -inline const uint8* WireFormatLite::ReadPrimitiveFromArray< - uint64, WireFormatLite::TYPE_FIXED64>( - const uint8* buffer, - uint64* value) { - return io::CodedInputStream::ReadLittleEndian64FromArray(buffer, value); -} -template <> -inline const uint8* WireFormatLite::ReadPrimitiveFromArray< - int32, WireFormatLite::TYPE_SFIXED32>( - const uint8* buffer, - int32* value) { - uint32 temp; - buffer = io::CodedInputStream::ReadLittleEndian32FromArray(buffer, &temp); - *value = static_cast(temp); - return buffer; -} -template <> -inline const uint8* WireFormatLite::ReadPrimitiveFromArray< - int64, WireFormatLite::TYPE_SFIXED64>( - const uint8* buffer, - int64* value) { - uint64 temp; - buffer = io::CodedInputStream::ReadLittleEndian64FromArray(buffer, &temp); - *value = static_cast(temp); - return buffer; -} -template <> -inline const uint8* WireFormatLite::ReadPrimitiveFromArray< - float, WireFormatLite::TYPE_FLOAT>( - const uint8* buffer, - float* value) { - uint32 temp; - buffer = io::CodedInputStream::ReadLittleEndian32FromArray(buffer, &temp); - *value = DecodeFloat(temp); - return buffer; -} -template <> -inline const uint8* WireFormatLite::ReadPrimitiveFromArray< - double, WireFormatLite::TYPE_DOUBLE>( - const uint8* buffer, - double* value) { - uint64 temp; - buffer = io::CodedInputStream::ReadLittleEndian64FromArray(buffer, &temp); - *value = DecodeDouble(temp); - return buffer; -} - -template -inline bool WireFormatLite::ReadRepeatedPrimitive( - int, // tag_size, unused. - uint32 tag, - io::CodedInputStream* input, - RepeatedField* values) { - CType value; - if (!ReadPrimitive(input, &value)) return false; - values->Add(value); - int elements_already_reserved = values->Capacity() - values->size(); - while (elements_already_reserved > 0 && input->ExpectTag(tag)) { - if (!ReadPrimitive(input, &value)) return false; - values->AddAlreadyReserved(value); - elements_already_reserved--; - } - return true; -} - -template -inline bool WireFormatLite::ReadRepeatedFixedSizePrimitive( - int tag_size, - uint32 tag, - io::CodedInputStream* input, - RepeatedField* values) { - GOOGLE_DCHECK_EQ(UInt32Size(tag), static_cast(tag_size)); - CType value; - if (!ReadPrimitive(input, &value)) - return false; - values->Add(value); - - // For fixed size values, repeated values can be read more quickly by - // reading directly from a raw array. - // - // We can get a tight loop by only reading as many elements as can be - // added to the RepeatedField without having to do any resizing. Additionally, - // we only try to read as many elements as are available from the current - // buffer space. Doing so avoids having to perform boundary checks when - // reading the value: the maximum number of elements that can be read is - // known outside of the loop. - const void* void_pointer; - int size; - input->GetDirectBufferPointerInline(&void_pointer, &size); - if (size > 0) { - const uint8* buffer = reinterpret_cast(void_pointer); - // The number of bytes each type occupies on the wire. - const int per_value_size = tag_size + static_cast(sizeof(value)); - - // parentheses around (std::min) prevents macro expansion of min(...) - int elements_available = - (std::min)(values->Capacity() - values->size(), size / per_value_size); - int num_read = 0; - while (num_read < elements_available && - (buffer = io::CodedInputStream::ExpectTagFromArray( - buffer, tag)) != NULL) { - buffer = ReadPrimitiveFromArray(buffer, &value); - values->AddAlreadyReserved(value); - ++num_read; - } - const int read_bytes = num_read * per_value_size; - if (read_bytes > 0) { - input->Skip(read_bytes); - } - } - return true; -} - -// Specializations of ReadRepeatedPrimitive for the fixed size types, which use -// the optimized code path. -#define READ_REPEATED_FIXED_SIZE_PRIMITIVE(CPPTYPE, DECLARED_TYPE) \ -template <> \ -inline bool WireFormatLite::ReadRepeatedPrimitive< \ - CPPTYPE, WireFormatLite::DECLARED_TYPE>( \ - int tag_size, \ - uint32 tag, \ - io::CodedInputStream* input, \ - RepeatedField* values) { \ - return ReadRepeatedFixedSizePrimitive< \ - CPPTYPE, WireFormatLite::DECLARED_TYPE>( \ - tag_size, tag, input, values); \ -} - -READ_REPEATED_FIXED_SIZE_PRIMITIVE(uint32, TYPE_FIXED32) -READ_REPEATED_FIXED_SIZE_PRIMITIVE(uint64, TYPE_FIXED64) -READ_REPEATED_FIXED_SIZE_PRIMITIVE(int32, TYPE_SFIXED32) -READ_REPEATED_FIXED_SIZE_PRIMITIVE(int64, TYPE_SFIXED64) -READ_REPEATED_FIXED_SIZE_PRIMITIVE(float, TYPE_FLOAT) -READ_REPEATED_FIXED_SIZE_PRIMITIVE(double, TYPE_DOUBLE) - -#undef READ_REPEATED_FIXED_SIZE_PRIMITIVE - -template -bool WireFormatLite::ReadRepeatedPrimitiveNoInline( - int tag_size, - uint32 tag, - io::CodedInputStream* input, - RepeatedField* value) { - return ReadRepeatedPrimitive( - tag_size, tag, input, value); -} - -template -inline bool WireFormatLite::ReadPackedPrimitive(io::CodedInputStream* input, - RepeatedField* values) { - int length; - if (!input->ReadVarintSizeAsInt(&length)) return false; - io::CodedInputStream::Limit limit = input->PushLimit(length); - while (input->BytesUntilLimit() > 0) { - CType value; - if (!ReadPrimitive(input, &value)) return false; - values->Add(value); - } - input->PopLimit(limit); - return true; -} - -template -inline bool WireFormatLite::ReadPackedFixedSizePrimitive( - io::CodedInputStream* input, RepeatedField* values) { - int length; - if (!input->ReadVarintSizeAsInt(&length)) return false; - const int old_entries = values->size(); - const int new_entries = length / static_cast(sizeof(CType)); - const int new_bytes = new_entries * static_cast(sizeof(CType)); - if (new_bytes != length) return false; - // We would *like* to pre-allocate the buffer to write into (for - // speed), but *must* avoid performing a very large allocation due - // to a malicious user-supplied "length" above. So we have a fast - // path that pre-allocates when the "length" is less than a bound. - // We determine the bound by calling BytesUntilTotalBytesLimit() and - // BytesUntilLimit(). These return -1 to mean "no limit set". - // There are four cases: - // TotalBytesLimit Limit - // -1 -1 Use slow path. - // -1 >= 0 Use fast path if length <= Limit. - // >= 0 -1 Use slow path. - // >= 0 >= 0 Use fast path if length <= min(both limits). - int64 bytes_limit = input->BytesUntilTotalBytesLimit(); - if (bytes_limit == -1) { - bytes_limit = input->BytesUntilLimit(); - } else { - // parentheses around (std::min) prevents macro expansion of min(...) - bytes_limit = - (std::min)(bytes_limit, static_cast(input->BytesUntilLimit())); - } - if (bytes_limit >= new_bytes) { - // Fast-path that pre-allocates *values to the final size. -#if defined(PROTOBUF_LITTLE_ENDIAN) - values->Resize(old_entries + new_entries, 0); - // values->mutable_data() may change after Resize(), so do this after: - void* dest = reinterpret_cast(values->mutable_data() + old_entries); - if (!input->ReadRaw(dest, new_bytes)) { - values->Truncate(old_entries); - return false; - } -#else - values->Reserve(old_entries + new_entries); - CType value; - for (int i = 0; i < new_entries; ++i) { - if (!ReadPrimitive(input, &value)) return false; - values->AddAlreadyReserved(value); - } -#endif - } else { - // This is the slow-path case where "length" may be too large to - // safely allocate. We read as much as we can into *values - // without pre-allocating "length" bytes. - CType value; - for (int i = 0; i < new_entries; ++i) { - if (!ReadPrimitive(input, &value)) return false; - values->Add(value); - } - } - return true; -} - -// Specializations of ReadPackedPrimitive for the fixed size types, which use -// an optimized code path. -#define READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(CPPTYPE, DECLARED_TYPE) \ -template <> \ -inline bool WireFormatLite::ReadPackedPrimitive< \ - CPPTYPE, WireFormatLite::DECLARED_TYPE>( \ - io::CodedInputStream* input, \ - RepeatedField* values) { \ - return ReadPackedFixedSizePrimitive< \ - CPPTYPE, WireFormatLite::DECLARED_TYPE>(input, values); \ -} - -READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(uint32, TYPE_FIXED32) -READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(uint64, TYPE_FIXED64) -READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(int32, TYPE_SFIXED32) -READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(int64, TYPE_SFIXED64) -READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(float, TYPE_FLOAT) -READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE(double, TYPE_DOUBLE) - -#undef READ_REPEATED_PACKED_FIXED_SIZE_PRIMITIVE - -template -bool WireFormatLite::ReadPackedPrimitiveNoInline(io::CodedInputStream* input, - RepeatedField* values) { - return ReadPackedPrimitive(input, values); -} - - -template -inline bool WireFormatLite::ReadGroup( - int field_number, io::CodedInputStream* input, - MessageType* value) { - if (!input->IncrementRecursionDepth()) return false; - if (!value->MergePartialFromCodedStream(input)) return false; - input->UnsafeDecrementRecursionDepth(); - // Make sure the last thing read was an end tag for this group. - if (!input->LastTagWas(MakeTag(field_number, WIRETYPE_END_GROUP))) { - return false; - } - return true; -} -template -inline bool WireFormatLite::ReadMessage( - io::CodedInputStream* input, MessageType* value) { - int length; - if (!input->ReadVarintSizeAsInt(&length)) return false; - std::pair p = - input->IncrementRecursionDepthAndPushLimit(length); - if (p.second < 0 || !value->MergePartialFromCodedStream(input)) return false; - // Make sure that parsing stopped when the limit was hit, not at an endgroup - // tag. - return input->DecrementRecursionDepthAndPopLimit(p.first); -} - -// =================================================================== - -inline void WireFormatLite::WriteTag(int field_number, WireType type, - io::CodedOutputStream* output) { - output->WriteTag(MakeTag(field_number, type)); -} - -inline void WireFormatLite::WriteInt32NoTag(int32 value, - io::CodedOutputStream* output) { - output->WriteVarint32SignExtended(value); -} -inline void WireFormatLite::WriteInt64NoTag(int64 value, - io::CodedOutputStream* output) { - output->WriteVarint64(static_cast(value)); -} -inline void WireFormatLite::WriteUInt32NoTag(uint32 value, - io::CodedOutputStream* output) { - output->WriteVarint32(value); -} -inline void WireFormatLite::WriteUInt64NoTag(uint64 value, - io::CodedOutputStream* output) { - output->WriteVarint64(value); -} -inline void WireFormatLite::WriteSInt32NoTag(int32 value, - io::CodedOutputStream* output) { - output->WriteVarint32(ZigZagEncode32(value)); -} -inline void WireFormatLite::WriteSInt64NoTag(int64 value, - io::CodedOutputStream* output) { - output->WriteVarint64(ZigZagEncode64(value)); -} -inline void WireFormatLite::WriteFixed32NoTag(uint32 value, - io::CodedOutputStream* output) { - output->WriteLittleEndian32(value); -} -inline void WireFormatLite::WriteFixed64NoTag(uint64 value, - io::CodedOutputStream* output) { - output->WriteLittleEndian64(value); -} -inline void WireFormatLite::WriteSFixed32NoTag(int32 value, - io::CodedOutputStream* output) { - output->WriteLittleEndian32(static_cast(value)); -} -inline void WireFormatLite::WriteSFixed64NoTag(int64 value, - io::CodedOutputStream* output) { - output->WriteLittleEndian64(static_cast(value)); -} -inline void WireFormatLite::WriteFloatNoTag(float value, - io::CodedOutputStream* output) { - output->WriteLittleEndian32(EncodeFloat(value)); -} -inline void WireFormatLite::WriteDoubleNoTag(double value, - io::CodedOutputStream* output) { - output->WriteLittleEndian64(EncodeDouble(value)); -} -inline void WireFormatLite::WriteBoolNoTag(bool value, - io::CodedOutputStream* output) { - output->WriteVarint32(value ? 1 : 0); -} -inline void WireFormatLite::WriteEnumNoTag(int value, - io::CodedOutputStream* output) { - output->WriteVarint32SignExtended(value); -} - -// See comment on ReadGroupNoVirtual to understand the need for this template -// parameter name. -template -inline void WireFormatLite::WriteGroupNoVirtual( - int field_number, const MessageType_WorkAroundCppLookupDefect& value, - io::CodedOutputStream* output) { - WriteTag(field_number, WIRETYPE_START_GROUP, output); - value.MessageType_WorkAroundCppLookupDefect::SerializeWithCachedSizes(output); - WriteTag(field_number, WIRETYPE_END_GROUP, output); -} -template -inline void WireFormatLite::WriteMessageNoVirtual( - int field_number, const MessageType_WorkAroundCppLookupDefect& value, - io::CodedOutputStream* output) { - WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); - output->WriteVarint32( - value.MessageType_WorkAroundCppLookupDefect::GetCachedSize()); - value.MessageType_WorkAroundCppLookupDefect::SerializeWithCachedSizes(output); -} - -// =================================================================== - -inline uint8* WireFormatLite::WriteTagToArray(int field_number, - WireType type, - uint8* target) { - return io::CodedOutputStream::WriteTagToArray(MakeTag(field_number, type), - target); -} - -inline uint8* WireFormatLite::WriteInt32NoTagToArray(int32 value, - uint8* target) { - return io::CodedOutputStream::WriteVarint32SignExtendedToArray(value, target); -} -inline uint8* WireFormatLite::WriteInt64NoTagToArray(int64 value, - uint8* target) { - return io::CodedOutputStream::WriteVarint64ToArray( - static_cast(value), target); -} -inline uint8* WireFormatLite::WriteUInt32NoTagToArray(uint32 value, - uint8* target) { - return io::CodedOutputStream::WriteVarint32ToArray(value, target); -} -inline uint8* WireFormatLite::WriteUInt64NoTagToArray(uint64 value, - uint8* target) { - return io::CodedOutputStream::WriteVarint64ToArray(value, target); -} -inline uint8* WireFormatLite::WriteSInt32NoTagToArray(int32 value, - uint8* target) { - return io::CodedOutputStream::WriteVarint32ToArray(ZigZagEncode32(value), - target); -} -inline uint8* WireFormatLite::WriteSInt64NoTagToArray(int64 value, - uint8* target) { - return io::CodedOutputStream::WriteVarint64ToArray(ZigZagEncode64(value), - target); -} -inline uint8* WireFormatLite::WriteFixed32NoTagToArray(uint32 value, - uint8* target) { - return io::CodedOutputStream::WriteLittleEndian32ToArray(value, target); -} -inline uint8* WireFormatLite::WriteFixed64NoTagToArray(uint64 value, - uint8* target) { - return io::CodedOutputStream::WriteLittleEndian64ToArray(value, target); -} -inline uint8* WireFormatLite::WriteSFixed32NoTagToArray(int32 value, - uint8* target) { - return io::CodedOutputStream::WriteLittleEndian32ToArray( - static_cast(value), target); -} -inline uint8* WireFormatLite::WriteSFixed64NoTagToArray(int64 value, - uint8* target) { - return io::CodedOutputStream::WriteLittleEndian64ToArray( - static_cast(value), target); -} -inline uint8* WireFormatLite::WriteFloatNoTagToArray(float value, - uint8* target) { - return io::CodedOutputStream::WriteLittleEndian32ToArray(EncodeFloat(value), - target); -} -inline uint8* WireFormatLite::WriteDoubleNoTagToArray(double value, - uint8* target) { - return io::CodedOutputStream::WriteLittleEndian64ToArray(EncodeDouble(value), - target); -} -inline uint8* WireFormatLite::WriteBoolNoTagToArray(bool value, - uint8* target) { - return io::CodedOutputStream::WriteVarint32ToArray(value ? 1 : 0, target); -} -inline uint8* WireFormatLite::WriteEnumNoTagToArray(int value, - uint8* target) { - return io::CodedOutputStream::WriteVarint32SignExtendedToArray(value, target); -} - -template -inline uint8* WireFormatLite::WritePrimitiveNoTagToArray( - const RepeatedField& value, - uint8* (*Writer)(T, uint8*), uint8* target) { - const int n = value.size(); - GOOGLE_DCHECK_GT(n, 0); - - const T* ii = value.unsafe_data(); - int i = 0; - do { - target = Writer(ii[i], target); - } while (++i < n); - - return target; -} - -template -inline uint8* WireFormatLite::WriteFixedNoTagToArray( - const RepeatedField& value, - uint8* (*Writer)(T, uint8*), uint8* target) { -#if defined(PROTOBUF_LITTLE_ENDIAN) - (void) Writer; - - const int n = value.size(); - GOOGLE_DCHECK_GT(n, 0); - - const T* ii = value.unsafe_data(); - const int bytes = n * static_cast(sizeof(ii[0])); - memcpy(target, ii, static_cast(bytes)); - return target + bytes; -#else - return WritePrimitiveNoTagToArray(value, Writer, target); -#endif -} - -inline uint8* WireFormatLite::WriteInt32NoTagToArray( - const RepeatedField< int32>& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteInt32NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteInt64NoTagToArray( - const RepeatedField< int64>& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteInt64NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteUInt32NoTagToArray( - const RepeatedField& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteUInt32NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteUInt64NoTagToArray( - const RepeatedField& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteUInt64NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteSInt32NoTagToArray( - const RepeatedField< int32>& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteSInt32NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteSInt64NoTagToArray( - const RepeatedField< int64>& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteSInt64NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteFixed32NoTagToArray( - const RepeatedField& value, uint8* target) { - return WriteFixedNoTagToArray(value, WriteFixed32NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteFixed64NoTagToArray( - const RepeatedField& value, uint8* target) { - return WriteFixedNoTagToArray(value, WriteFixed64NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteSFixed32NoTagToArray( - const RepeatedField< int32>& value, uint8* target) { - return WriteFixedNoTagToArray(value, WriteSFixed32NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteSFixed64NoTagToArray( - const RepeatedField< int64>& value, uint8* target) { - return WriteFixedNoTagToArray(value, WriteSFixed64NoTagToArray, target); -} -inline uint8* WireFormatLite::WriteFloatNoTagToArray( - const RepeatedField< float>& value, uint8* target) { - return WriteFixedNoTagToArray(value, WriteFloatNoTagToArray, target); -} -inline uint8* WireFormatLite::WriteDoubleNoTagToArray( - const RepeatedField& value, uint8* target) { - return WriteFixedNoTagToArray(value, WriteDoubleNoTagToArray, target); -} -inline uint8* WireFormatLite::WriteBoolNoTagToArray( - const RepeatedField< bool>& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteBoolNoTagToArray, target); -} -inline uint8* WireFormatLite::WriteEnumNoTagToArray( - const RepeatedField< int>& value, uint8* target) { - return WritePrimitiveNoTagToArray(value, WriteEnumNoTagToArray, target); -} - -inline uint8* WireFormatLite::WriteInt32ToArray(int field_number, - int32 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteInt32NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteInt64ToArray(int field_number, - int64 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteInt64NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteUInt32ToArray(int field_number, - uint32 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteUInt32NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteUInt64ToArray(int field_number, - uint64 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteUInt64NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteSInt32ToArray(int field_number, - int32 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteSInt32NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteSInt64ToArray(int field_number, - int64 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteSInt64NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteFixed32ToArray(int field_number, - uint32 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_FIXED32, target); - return WriteFixed32NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteFixed64ToArray(int field_number, - uint64 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_FIXED64, target); - return WriteFixed64NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteSFixed32ToArray(int field_number, - int32 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_FIXED32, target); - return WriteSFixed32NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteSFixed64ToArray(int field_number, - int64 value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_FIXED64, target); - return WriteSFixed64NoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteFloatToArray(int field_number, - float value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_FIXED32, target); - return WriteFloatNoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteDoubleToArray(int field_number, - double value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_FIXED64, target); - return WriteDoubleNoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteBoolToArray(int field_number, - bool value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteBoolNoTagToArray(value, target); -} -inline uint8* WireFormatLite::WriteEnumToArray(int field_number, - int value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_VARINT, target); - return WriteEnumNoTagToArray(value, target); -} - -template -inline uint8* WireFormatLite::WritePrimitiveToArray( - int field_number, - const RepeatedField& value, - uint8* (*Writer)(int, T, uint8*), uint8* target) { - const int n = value.size(); - if (n == 0) { - return target; - } - - const T* ii = value.unsafe_data(); - int i = 0; - do { - target = Writer(field_number, ii[i], target); - } while (++i < n); - - return target; -} - -inline uint8* WireFormatLite::WriteInt32ToArray( - int field_number, const RepeatedField< int32>& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteInt32ToArray, target); -} -inline uint8* WireFormatLite::WriteInt64ToArray( - int field_number, const RepeatedField< int64>& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteInt64ToArray, target); -} -inline uint8* WireFormatLite::WriteUInt32ToArray( - int field_number, const RepeatedField& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteUInt32ToArray, target); -} -inline uint8* WireFormatLite::WriteUInt64ToArray( - int field_number, const RepeatedField& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteUInt64ToArray, target); -} -inline uint8* WireFormatLite::WriteSInt32ToArray( - int field_number, const RepeatedField< int32>& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteSInt32ToArray, target); -} -inline uint8* WireFormatLite::WriteSInt64ToArray( - int field_number, const RepeatedField< int64>& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteSInt64ToArray, target); -} -inline uint8* WireFormatLite::WriteFixed32ToArray( - int field_number, const RepeatedField& value, uint8* target) { - return WritePrimitiveToArray( - field_number, value, WriteFixed32ToArray, target); -} -inline uint8* WireFormatLite::WriteFixed64ToArray( - int field_number, const RepeatedField& value, uint8* target) { - return WritePrimitiveToArray( - field_number, value, WriteFixed64ToArray, target); -} -inline uint8* WireFormatLite::WriteSFixed32ToArray( - int field_number, const RepeatedField< int32>& value, uint8* target) { - return WritePrimitiveToArray( - field_number, value, WriteSFixed32ToArray, target); -} -inline uint8* WireFormatLite::WriteSFixed64ToArray( - int field_number, const RepeatedField< int64>& value, uint8* target) { - return WritePrimitiveToArray( - field_number, value, WriteSFixed64ToArray, target); -} -inline uint8* WireFormatLite::WriteFloatToArray( - int field_number, const RepeatedField< float>& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteFloatToArray, target); -} -inline uint8* WireFormatLite::WriteDoubleToArray( - int field_number, const RepeatedField& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteDoubleToArray, target); -} -inline uint8* WireFormatLite::WriteBoolToArray( - int field_number, const RepeatedField< bool>& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteBoolToArray, target); -} -inline uint8* WireFormatLite::WriteEnumToArray( - int field_number, const RepeatedField< int>& value, uint8* target) { - return WritePrimitiveToArray(field_number, value, WriteEnumToArray, target); -} -inline uint8* WireFormatLite::WriteStringToArray(int field_number, - const string& value, - uint8* target) { - // String is for UTF-8 text only - // WARNING: In wire_format.cc, both strings and bytes are handled by - // WriteString() to avoid code duplication. If the implementations become - // different, you will need to update that usage. - target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); - return io::CodedOutputStream::WriteStringWithSizeToArray(value, target); -} -inline uint8* WireFormatLite::WriteBytesToArray(int field_number, - const string& value, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); - return io::CodedOutputStream::WriteStringWithSizeToArray(value, target); -} - - -template -inline uint8* WireFormatLite::InternalWriteGroupToArray( - int field_number, const MessageType& value, bool deterministic, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_START_GROUP, target); - target = value.InternalSerializeWithCachedSizesToArray(deterministic, target); - return WriteTagToArray(field_number, WIRETYPE_END_GROUP, target); -} -template -inline uint8* WireFormatLite::InternalWriteMessageToArray( - int field_number, const MessageType& value, bool deterministic, - uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); - target = io::CodedOutputStream::WriteVarint32ToArray( - static_cast(value.GetCachedSize()), target); - return value.InternalSerializeWithCachedSizesToArray(deterministic, target); -} - -// See comment on ReadGroupNoVirtual to understand the need for this template -// parameter name. -template -inline uint8* WireFormatLite::InternalWriteGroupNoVirtualToArray( - int field_number, const MessageType_WorkAroundCppLookupDefect& value, - bool deterministic, uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_START_GROUP, target); - target = value.MessageType_WorkAroundCppLookupDefect:: - InternalSerializeWithCachedSizesToArray(deterministic, target); - return WriteTagToArray(field_number, WIRETYPE_END_GROUP, target); -} -template -inline uint8* WireFormatLite::InternalWriteMessageNoVirtualToArray( - int field_number, const MessageType_WorkAroundCppLookupDefect& value, - bool deterministic, uint8* target) { - target = WriteTagToArray(field_number, WIRETYPE_LENGTH_DELIMITED, target); - target = io::CodedOutputStream::WriteVarint32ToArray( - static_cast( - value.MessageType_WorkAroundCppLookupDefect::GetCachedSize()), - target); - return value.MessageType_WorkAroundCppLookupDefect:: - InternalSerializeWithCachedSizesToArray(deterministic, target); -} - -// =================================================================== - -inline size_t WireFormatLite::Int32Size(int32 value) { - return io::CodedOutputStream::VarintSize32SignExtended(value); -} -inline size_t WireFormatLite::Int64Size(int64 value) { - return io::CodedOutputStream::VarintSize64(static_cast(value)); -} -inline size_t WireFormatLite::UInt32Size(uint32 value) { - return io::CodedOutputStream::VarintSize32(value); -} -inline size_t WireFormatLite::UInt64Size(uint64 value) { - return io::CodedOutputStream::VarintSize64(value); -} -inline size_t WireFormatLite::SInt32Size(int32 value) { - return io::CodedOutputStream::VarintSize32(ZigZagEncode32(value)); -} -inline size_t WireFormatLite::SInt64Size(int64 value) { - return io::CodedOutputStream::VarintSize64(ZigZagEncode64(value)); -} -inline size_t WireFormatLite::EnumSize(int value) { - return io::CodedOutputStream::VarintSize32SignExtended(value); -} - -inline size_t WireFormatLite::StringSize(const string& value) { - return LengthDelimitedSize(value.size()); -} -inline size_t WireFormatLite::BytesSize(const string& value) { - return LengthDelimitedSize(value.size()); -} - - -template -inline size_t WireFormatLite::GroupSize(const MessageType& value) { - return value.ByteSizeLong(); -} -template -inline size_t WireFormatLite::MessageSize(const MessageType& value) { - return LengthDelimitedSize(value.ByteSizeLong()); -} - -// See comment on ReadGroupNoVirtual to understand the need for this template -// parameter name. -template -inline size_t WireFormatLite::GroupSizeNoVirtual( - const MessageType_WorkAroundCppLookupDefect& value) { - return value.MessageType_WorkAroundCppLookupDefect::ByteSizeLong(); -} -template -inline size_t WireFormatLite::MessageSizeNoVirtual( - const MessageType_WorkAroundCppLookupDefect& value) { - return LengthDelimitedSize( - value.MessageType_WorkAroundCppLookupDefect::ByteSizeLong()); -} - -inline size_t WireFormatLite::LengthDelimitedSize(size_t length) { - // The static_cast here prevents an error in certain compiler configurations - // but is not technically correct--if length is too large to fit in a uint32 - // then it will be silently truncated. We will need to fix this if we ever - // decide to start supporting serialized messages greater than 2 GiB in size. - return length + io::CodedOutputStream::VarintSize32( - static_cast(length)); -} - -} // namespace internal -} // namespace protobuf - -} // namespace google -#endif // GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__ diff --git a/third_party/protobuf-lite/implicit_weak_message.cc b/third_party/protobuf-lite/implicit_weak_message.cc index 7a1d4446..53906169 100644 --- a/third_party/protobuf-lite/implicit_weak_message.cc +++ b/third_party/protobuf-lite/implicit_weak_message.cc @@ -30,31 +30,33 @@ #include -#include +#include #include +#include #include +#include + namespace google { namespace protobuf { namespace internal { -bool ImplicitWeakMessage::MergePartialFromCodedStream(io::CodedInputStream* input) { - io::StringOutputStream string_stream(&data_); - io::CodedOutputStream coded_stream(&string_stream, false); - return WireFormatLite::SkipMessage(input, &coded_stream); +const char* ImplicitWeakMessage::_InternalParse(const char* ptr, + ParseContext* ctx) { + return ctx->AppendString(ptr, &data_); } -::google::protobuf::internal::ExplicitlyConstructed +ExplicitlyConstructed implicit_weak_message_default_instance; -GOOGLE_PROTOBUF_DECLARE_ONCE(implicit_weak_message_once_init_); +internal::once_flag implicit_weak_message_once_init_; void InitImplicitWeakMessageDefaultInstance() { implicit_weak_message_default_instance.DefaultConstruct(); } const ImplicitWeakMessage* ImplicitWeakMessage::default_instance() { - ::google::protobuf::GoogleOnceInit(&implicit_weak_message_once_init_, - &InitImplicitWeakMessageDefaultInstance); + internal::call_once(implicit_weak_message_once_init_, + InitImplicitWeakMessageDefaultInstance); return &implicit_weak_message_default_instance.get(); } diff --git a/third_party/protobuf-lite/int128.cc b/third_party/protobuf-lite/int128.cc index a5090801..2119e655 100644 --- a/third_party/protobuf-lite/int128.cc +++ b/third_party/protobuf-lite/int128.cc @@ -34,12 +34,16 @@ #include // NOLINT(readability/streams) #include +#include + +#include + namespace google { namespace protobuf { const uint128_pod kuint128max = { - static_cast(GOOGLE_LONGLONG(0xFFFFFFFFFFFFFFFF)), - static_cast(GOOGLE_LONGLONG(0xFFFFFFFFFFFFFFFF)) + static_cast(PROTOBUF_LONGLONG(0xFFFFFFFFFFFFFFFF)), + static_cast(PROTOBUF_LONGLONG(0xFFFFFFFFFFFFFFFF)) }; // Returns the 0-based position of the last set bit (i.e., most significant bit) @@ -63,7 +67,7 @@ static inline int Fls64(uint64 n) { STEP(uint32, n32, pos, 0x10); STEP(uint32, n32, pos, 0x08); STEP(uint32, n32, pos, 0x04); - return pos + ((GOOGLE_ULONGLONG(0x3333333322221100) >> (n32 << 2)) & 0x3); + return pos + ((PROTOBUF_ULONGLONG(0x3333333322221100) >> (n32 << 2)) & 0x3); } #undef STEP @@ -76,52 +80,36 @@ static inline int Fls128(uint128 n) { return Fls64(Uint128Low64(n)); } -// Long division/modulo for uint128 implemented using the shift-subtract -// division algorithm adapted from: -// http://stackoverflow.com/questions/5386377/division-without-using void uint128::DivModImpl(uint128 dividend, uint128 divisor, uint128* quotient_ret, uint128* remainder_ret) { if (divisor == 0) { GOOGLE_LOG(FATAL) << "Division or mod by zero: dividend.hi=" << dividend.hi_ << ", lo=" << dividend.lo_; - } - - if (divisor > dividend) { + } else if (dividend < divisor) { *quotient_ret = 0; *remainder_ret = dividend; return; - } - - if (divisor == dividend) { - *quotient_ret = 1; - *remainder_ret = 0; - return; - } - - uint128 denominator = divisor; - uint128 position = 1; - uint128 quotient = 0; - - // Left aligns the MSB of the denominator and the dividend. - int shift = Fls128(dividend) - Fls128(denominator); - denominator <<= shift; - position <<= shift; - - // Uses shift-subtract algorithm to divide dividend by denominator. The - // remainder will be left in dividend. - while (position > 0) { - if (dividend >= denominator) { - dividend -= denominator; - quotient |= position; + } else { + int dividend_bit_length = Fls128(dividend); + int divisor_bit_length = Fls128(divisor); + int difference = dividend_bit_length - divisor_bit_length; + uint128 quotient = 0; + while (difference >= 0) { + quotient <<= 1; + uint128 shifted_divisor = divisor << difference; + if (shifted_divisor <= dividend) { + dividend -= shifted_divisor; + quotient += 1; + } + difference -= 1; } - position >>= 1; - denominator >>= 1; + //record the final quotient and remainder + *quotient_ret = quotient; + *remainder_ret = dividend; } - - *quotient_ret = quotient; - *remainder_ret = dividend; } + uint128& uint128::operator/=(const uint128& divisor) { uint128 quotient = 0; uint128 remainder = 0; @@ -145,15 +133,18 @@ std::ostream& operator<<(std::ostream& o, const uint128& b) { std::streamsize div_base_log; switch (flags & std::ios::basefield) { case std::ios::hex: - div = static_cast(GOOGLE_ULONGLONG(0x1000000000000000)); // 16^15 + div = + static_cast(PROTOBUF_ULONGLONG(0x1000000000000000)); // 16^15 div_base_log = 15; break; case std::ios::oct: - div = static_cast(GOOGLE_ULONGLONG(01000000000000000000000)); // 8^21 + div = static_cast( + PROTOBUF_ULONGLONG(01000000000000000000000)); // 8^21 div_base_log = 21; break; default: // std::ios::dec - div = static_cast(GOOGLE_ULONGLONG(10000000000000000000)); // 10^19 + div = static_cast( + PROTOBUF_ULONGLONG(10000000000000000000)); // 10^19 div_base_log = 19; break; } diff --git a/third_party/protobuf-lite/io_win32.cc b/third_party/protobuf-lite/io_win32.cc index 4407facb..d22ceac7 100644 --- a/third_party/protobuf-lite/io_win32.cc +++ b/third_party/protobuf-lite/io_win32.cc @@ -29,7 +29,9 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Author: laszlocsomor@google.com (Laszlo Csomor) -// +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + // Implementation for long-path-aware open/mkdir/access/etc. on Windows, as well // as for the supporting utility functions. // @@ -40,25 +42,29 @@ // // This file is only used on Windows, it's empty on other platforms. -#if defined(_WIN32) +#if defined(_WIN32) && !defined(_XBOX_ONE) // Comment this out to fall back to using the ANSI versions (open, mkdir, ...) // instead of the Unicode ones (_wopen, _wmkdir, ...). Doing so can be useful to // debug failing tests if that's caused by the long path support. #define SUPPORT_LONGPATHS +#include + #include #include #include #include #include -#include #include #include #include -#include -#include +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN 1 +#endif + +#include #include #include @@ -67,7 +73,7 @@ namespace google { namespace protobuf { -namespace internal { +namespace io { namespace win32 { namespace { @@ -91,7 +97,7 @@ struct CharTraits { template bool null_or_empty(const char_type* s) { - return s == NULL || *s == 0; + return s == nullptr || *s == 0; } // Returns true if the path starts with a drive letter, e.g. "c:". @@ -225,7 +231,7 @@ bool as_windows_path(const char* path, wstring* result) { if (!is_path_absolute(wpath.c_str())) { - int size = ::GetCurrentDirectoryW(0, NULL); + int size = ::GetCurrentDirectoryW(0, nullptr); if (size == 0 && GetLastError() != ERROR_INSUFFICIENT_BUFFER) { return false; } @@ -316,17 +322,17 @@ FILE* fopen(const char* path, const char* mode) { #ifdef SUPPORT_LONGPATHS if (null_or_empty(path)) { errno = EINVAL; - return NULL; + return nullptr; } wstring wpath; if (!as_windows_path(path, &wpath)) { errno = ENOENT; - return NULL; + return nullptr; } wstring wmode; if (!strings::utf8_to_wcs(mode, &wmode)) { errno = EINVAL; - return NULL; + return nullptr; } return ::_wfopen(wpath.c_str(), wmode.c_str()); #else @@ -334,7 +340,7 @@ FILE* fopen(const char* path, const char* mode) { #endif } -int close(int fd) { return ::close(fd); } +int close(int fd) { return ::_close(fd); } int dup(int fd) { return ::_dup(fd); } @@ -355,6 +361,56 @@ wstring testonly_utf8_to_winpath(const char* path) { return as_windows_path(path, &wpath) ? wpath : wstring(); } +ExpandWildcardsResult ExpandWildcards( + const string& path, std::function consume) { + if (path.find_first_of("*?") == string::npos) { + // There are no wildcards in the path, we don't need to expand it. + consume(path); + return ExpandWildcardsResult::kSuccess; + } + + wstring wpath; + if (!as_windows_path(path.c_str(), &wpath)) { + return ExpandWildcardsResult::kErrorInputPathConversion; + } + + static const wstring kDot = L"."; + static const wstring kDotDot = L".."; + WIN32_FIND_DATAW metadata; + HANDLE handle = ::FindFirstFileW(wpath.c_str(), &metadata); + if (handle == INVALID_HANDLE_VALUE) { + // The pattern does not match any files (or directories). + return ExpandWildcardsResult::kErrorNoMatchingFile; + } + + string::size_type pos = path.find_last_of("\\/"); + string dirname; + if (pos != string::npos) { + dirname = path.substr(0, pos + 1); + } + + ExpandWildcardsResult matched = ExpandWildcardsResult::kErrorNoMatchingFile; + do { + // Ignore ".", "..", and directories. + if ((metadata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 && + kDot != metadata.cFileName && kDotDot != metadata.cFileName) { + matched = ExpandWildcardsResult::kSuccess; + string filename; + if (!strings::wcs_to_utf8(metadata.cFileName, &filename)) { + return ExpandWildcardsResult::kErrorOutputPathConversion; + } + + if (dirname.empty()) { + consume(filename); + } else { + consume(dirname + filename); + } + } + } while (::FindNextFileW(handle, &metadata)); + FindClose(handle); + return matched; +} + namespace strings { bool wcs_to_mbs(const WCHAR* s, string* out, bool outUtf8) { @@ -365,15 +421,15 @@ bool wcs_to_mbs(const WCHAR* s, string* out, bool outUtf8) { BOOL usedDefaultChar = FALSE; SetLastError(0); int size = WideCharToMultiByte( - outUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, NULL, 0, NULL, - outUtf8 ? NULL : &usedDefaultChar); + outUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, nullptr, 0, nullptr, + outUtf8 ? nullptr : &usedDefaultChar); if ((size == 0 && GetLastError() != ERROR_INSUFFICIENT_BUFFER) || usedDefaultChar) { return false; } std::unique_ptr astr(new CHAR[size]); WideCharToMultiByte( - outUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, astr.get(), size, NULL, NULL); + outUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, astr.get(), size, nullptr, nullptr); out->assign(astr.get()); return true; } @@ -386,7 +442,7 @@ bool mbs_to_wcs(const char* s, wstring* out, bool inUtf8) { SetLastError(0); int size = - MultiByteToWideChar(inUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, NULL, 0); + MultiByteToWideChar(inUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, nullptr, 0); if (size == 0 && GetLastError() != ERROR_INSUFFICIENT_BUFFER) { return false; } @@ -407,7 +463,7 @@ bool wcs_to_utf8(const wchar_t* input, string* out) { } // namespace strings } // namespace win32 -} // namespace internal +} // namespace io } // namespace protobuf } // namespace google diff --git a/third_party/protobuf-lite/message_lite.cc b/third_party/protobuf-lite/message_lite.cc index 123b142d..5b1db7b9 100644 --- a/third_party/protobuf-lite/message_lite.cc +++ b/third_party/protobuf-lite/message_lite.cc @@ -33,26 +33,42 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include - -#include -#include #include -#include + +#include +#include #include + #include #include +#include +#include #include +#include +#include #include +#include +#include +#include +#include +#include #include +#include + +#include namespace google { namespace protobuf { -string MessageLite::InitializationErrorString() const { +std::string MessageLite::InitializationErrorString() const { return "(cannot determine missing fields for lite message)"; } +std::string MessageLite::DebugString() const { + std::uintptr_t address = reinterpret_cast(this); + return StrCat("MessageLite at 0x", strings::Hex(address)); +} + namespace { // When serializing, we first compute the byte size, then serialize the message. @@ -71,12 +87,13 @@ void ByteSizeConsistencyError(size_t byte_size_before_serialization, GOOGLE_CHECK_EQ(bytes_produced_by_serialization, byte_size_before_serialization) << "Byte size calculation and serialization were inconsistent. This " "may indicate a bug in protocol buffers or it may be caused by " - "concurrent modification of " << message.GetTypeName() << "."; + "concurrent modification of " + << message.GetTypeName() << "."; GOOGLE_LOG(FATAL) << "This shouldn't be called if all the sizes are equal."; } -string InitializationErrorMessage(const char* action, - const MessageLite& message) { +std::string InitializationErrorMessage(const char* action, + const MessageLite& message) { // Note: We want to avoid depending on strutil in the lite library, otherwise // we'd use: // @@ -86,7 +103,7 @@ string InitializationErrorMessage(const char* action, // action, message.GetTypeName(), // message.InitializationErrorString()); - string result; + std::string result; result += "Can't "; result += action; result += " message of type \""; @@ -96,64 +113,89 @@ string InitializationErrorMessage(const char* action, return result; } -// Several of the Parse methods below just do one thing and then call another -// method. In a naive implementation, we might have ParseFromString() call -// ParseFromArray() which would call ParseFromZeroCopyStream() which would call -// ParseFromCodedStream() which would call MergeFromCodedStream() which would -// call MergePartialFromCodedStream(). However, when parsing very small -// messages, every function call introduces significant overhead. To avoid -// this without reproducing code, we use these forced-inline helpers. -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE bool InlineMergeFromCodedStream( - io::CodedInputStream* input, MessageLite* message); -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE bool InlineParseFromCodedStream( - io::CodedInputStream* input, MessageLite* message); -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE bool InlineParsePartialFromCodedStream( - io::CodedInputStream* input, MessageLite* message); -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE bool InlineParseFromArray( - const void* data, int size, MessageLite* message); -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE bool InlineParsePartialFromArray( - const void* data, int size, MessageLite* message); - -inline bool InlineMergeFromCodedStream(io::CodedInputStream* input, - MessageLite* message) { - if (!message->MergePartialFromCodedStream(input)) return false; - if (!message->IsInitialized()) { - GOOGLE_LOG(ERROR) << InitializationErrorMessage("parse", *message); - return false; - } - return true; +inline StringPiece as_string_view(const void* data, int size) { + return StringPiece(static_cast(data), size); } -inline bool InlineParseFromCodedStream(io::CodedInputStream* input, - MessageLite* message) { - message->Clear(); - return InlineMergeFromCodedStream(input, message); +// Returns true of all required fields are present / have values. +inline bool CheckFieldPresence(const internal::ParseContext& ctx, + const MessageLite& msg, + MessageLite::ParseFlags parse_flags) { + if (PROTOBUF_PREDICT_FALSE((parse_flags & MessageLite::kMergePartial) != 0)) { + return true; + } + return msg.IsInitializedWithErrors(); } -inline bool InlineParsePartialFromCodedStream(io::CodedInputStream* input, - MessageLite* message) { - message->Clear(); - return message->MergePartialFromCodedStream(input); -} +} // namespace -inline bool InlineParseFromArray( - const void* data, int size, MessageLite* message) { - io::CodedInputStream input(reinterpret_cast(data), size); - return InlineParseFromCodedStream(&input, message) && - input.ConsumedEntireMessage(); +void MessageLite::LogInitializationErrorMessage() const { + GOOGLE_LOG(ERROR) << InitializationErrorMessage("parse", *this); } -inline bool InlineParsePartialFromArray( - const void* data, int size, MessageLite* message) { - io::CodedInputStream input(reinterpret_cast(data), size); - return InlineParsePartialFromCodedStream(&input, message) && - input.ConsumedEntireMessage(); -} +namespace internal { -} // namespace +template +bool MergeFromImpl(StringPiece input, MessageLite* msg, + MessageLite::ParseFlags parse_flags) { + const char* ptr; + internal::ParseContext ctx(io::CodedInputStream::GetDefaultRecursionLimit(), + aliasing, &ptr, input); + ptr = msg->_InternalParse(ptr, &ctx); + // ctx has an explicit limit set (length of string_view). + if (PROTOBUF_PREDICT_TRUE(ptr && ctx.EndedAtLimit())) { + return CheckFieldPresence(ctx, *msg, parse_flags); + } + return false; +} + +template +bool MergeFromImpl(io::ZeroCopyInputStream* input, MessageLite* msg, + MessageLite::ParseFlags parse_flags) { + const char* ptr; + internal::ParseContext ctx(io::CodedInputStream::GetDefaultRecursionLimit(), + aliasing, &ptr, input); + ptr = msg->_InternalParse(ptr, &ctx); + // ctx has no explicit limit (hence we end on end of stream) + if (PROTOBUF_PREDICT_TRUE(ptr && ctx.EndedAtEndOfStream())) { + return CheckFieldPresence(ctx, *msg, parse_flags); + } + return false; +} + +template +bool MergeFromImpl(BoundedZCIS input, MessageLite* msg, + MessageLite::ParseFlags parse_flags) { + const char* ptr; + internal::ParseContext ctx(io::CodedInputStream::GetDefaultRecursionLimit(), + aliasing, &ptr, input.zcis, input.limit); + ptr = msg->_InternalParse(ptr, &ctx); + if (PROTOBUF_PREDICT_FALSE(!ptr)) return false; + ctx.BackUp(ptr); + if (PROTOBUF_PREDICT_TRUE(ctx.EndedAtLimit())) { + return CheckFieldPresence(ctx, *msg, parse_flags); + } + return false; +} + +template bool MergeFromImpl(StringPiece input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +template bool MergeFromImpl(StringPiece input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +template bool MergeFromImpl(io::ZeroCopyInputStream* input, + MessageLite* msg, + MessageLite::ParseFlags parse_flags); +template bool MergeFromImpl(io::ZeroCopyInputStream* input, + MessageLite* msg, + MessageLite::ParseFlags parse_flags); +template bool MergeFromImpl(BoundedZCIS input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +template bool MergeFromImpl(BoundedZCIS input, MessageLite* msg, + MessageLite::ParseFlags parse_flags); +} // namespace internal -MessageLite* MessageLite::New(::google::protobuf::Arena* arena) const { +MessageLite* MessageLite::New(Arena* arena) const { MessageLite* message = New(); if (arena != NULL) { arena->Own(message); @@ -161,70 +203,169 @@ MessageLite* MessageLite::New(::google::protobuf::Arena* arena) const { return message; } +class ZeroCopyCodedInputStream : public io::ZeroCopyInputStream { + public: + ZeroCopyCodedInputStream(io::CodedInputStream* cis) : cis_(cis) {} + bool Next(const void** data, int* size) final { + if (!cis_->GetDirectBufferPointer(data, size)) return false; + cis_->Skip(*size); + return true; + } + void BackUp(int count) final { cis_->Advance(-count); } + bool Skip(int count) final { return cis_->Skip(count); } + int64_t ByteCount() const final { return 0; } + + bool aliasing_enabled() { return cis_->aliasing_enabled_; } + + private: + io::CodedInputStream* cis_; +}; + +bool MessageLite::MergeFromImpl(io::CodedInputStream* input, + MessageLite::ParseFlags parse_flags) { + ZeroCopyCodedInputStream zcis(input); + const char* ptr; + internal::ParseContext ctx(input->RecursionBudget(), zcis.aliasing_enabled(), + &ptr, &zcis); + // MergePartialFromCodedStream allows terminating the wireformat by 0 or + // end-group tag. Leaving it up to the caller to verify correct ending by + // calling LastTagWas on input. We need to maintain this behavior. + ctx.TrackCorrectEnding(); + ctx.data().pool = input->GetExtensionPool(); + ctx.data().factory = input->GetExtensionFactory(); + ptr = _InternalParse(ptr, &ctx); + if (PROTOBUF_PREDICT_FALSE(!ptr)) return false; + ctx.BackUp(ptr); + if (!ctx.EndedAtEndOfStream()) { + GOOGLE_DCHECK(ctx.LastTag() != 1); // We can't end on a pushed limit. + if (ctx.IsExceedingLimit(ptr)) return false; + input->SetLastTag(ctx.LastTag()); + } else { + input->SetConsumed(); + } + return CheckFieldPresence(ctx, *this, parse_flags); +} + +bool MessageLite::MergePartialFromCodedStream(io::CodedInputStream* input) { + return MergeFromImpl(input, kMergePartial); +} + bool MessageLite::MergeFromCodedStream(io::CodedInputStream* input) { - return InlineMergeFromCodedStream(input, this); + return MergeFromImpl(input, kMerge); } bool MessageLite::ParseFromCodedStream(io::CodedInputStream* input) { - return InlineParseFromCodedStream(input, this); + Clear(); + return MergeFromImpl(input, kParse); } bool MessageLite::ParsePartialFromCodedStream(io::CodedInputStream* input) { - return InlineParsePartialFromCodedStream(input, this); + Clear(); + return MergeFromImpl(input, kParsePartial); } bool MessageLite::ParseFromZeroCopyStream(io::ZeroCopyInputStream* input) { - io::CodedInputStream decoder(input); - return ParseFromCodedStream(&decoder) && decoder.ConsumedEntireMessage(); + return ParseFrom(input); } bool MessageLite::ParsePartialFromZeroCopyStream( io::ZeroCopyInputStream* input) { - io::CodedInputStream decoder(input); - return ParsePartialFromCodedStream(&decoder) && - decoder.ConsumedEntireMessage(); + return ParseFrom(input); +} + +bool MessageLite::ParseFromFileDescriptor(int file_descriptor) { + io::FileInputStream input(file_descriptor); + return ParseFromZeroCopyStream(&input) && input.GetErrno() == 0; +} + +bool MessageLite::ParsePartialFromFileDescriptor(int file_descriptor) { + io::FileInputStream input(file_descriptor); + return ParsePartialFromZeroCopyStream(&input) && input.GetErrno() == 0; +} + +bool MessageLite::ParseFromIstream(std::istream* input) { + io::IstreamInputStream zero_copy_input(input); + return ParseFromZeroCopyStream(&zero_copy_input) && input->eof(); } -bool MessageLite::ParseFromBoundedZeroCopyStream( +bool MessageLite::ParsePartialFromIstream(std::istream* input) { + io::IstreamInputStream zero_copy_input(input); + return ParsePartialFromZeroCopyStream(&zero_copy_input) && input->eof(); +} + +bool MessageLite::MergePartialFromBoundedZeroCopyStream( io::ZeroCopyInputStream* input, int size) { - io::CodedInputStream decoder(input); - decoder.PushLimit(size); - return ParseFromCodedStream(&decoder) && - decoder.ConsumedEntireMessage() && - decoder.BytesUntilLimit() == 0; + return ParseFrom(internal::BoundedZCIS{input, size}); +} + +bool MessageLite::MergeFromBoundedZeroCopyStream(io::ZeroCopyInputStream* input, + int size) { + return ParseFrom(internal::BoundedZCIS{input, size}); +} + +bool MessageLite::ParseFromBoundedZeroCopyStream(io::ZeroCopyInputStream* input, + int size) { + return ParseFrom(internal::BoundedZCIS{input, size}); } bool MessageLite::ParsePartialFromBoundedZeroCopyStream( io::ZeroCopyInputStream* input, int size) { - io::CodedInputStream decoder(input); - decoder.PushLimit(size); - return ParsePartialFromCodedStream(&decoder) && - decoder.ConsumedEntireMessage() && - decoder.BytesUntilLimit() == 0; + return ParseFrom(internal::BoundedZCIS{input, size}); } -bool MessageLite::ParseFromString(const string& data) { - return InlineParseFromArray(data.data(), data.size(), this); +bool MessageLite::ParseFromString(const std::string& data) { + return ParseFrom(data); } -bool MessageLite::ParsePartialFromString(const string& data) { - return InlineParsePartialFromArray(data.data(), data.size(), this); +bool MessageLite::ParsePartialFromString(const std::string& data) { + return ParseFrom(data); } bool MessageLite::ParseFromArray(const void* data, int size) { - return InlineParseFromArray(data, size, this); + return ParseFrom(as_string_view(data, size)); } bool MessageLite::ParsePartialFromArray(const void* data, int size) { - return InlineParsePartialFromArray(data, size, this); + return ParseFrom(as_string_view(data, size)); +} + +bool MessageLite::MergeFromString(const std::string& data) { + return ParseFrom(data); } // =================================================================== +inline uint8* SerializeToArrayImpl(const MessageLite& msg, uint8* target, + int size) { + constexpr bool debug = false; + if (debug) { + // Force serialization to a stream with a block size of 1, which forces + // all writes to the stream to cross buffers triggering all fallback paths + // in the unittests when serializing to string / array. + io::ArrayOutputStream stream(target, size, 1); + uint8* ptr; + io::EpsCopyOutputStream out( + &stream, io::CodedOutputStream::IsDefaultSerializationDeterministic(), + &ptr); + ptr = msg._InternalSerialize(ptr, &out); + out.Trim(ptr); + GOOGLE_DCHECK(!out.HadError() && stream.ByteCount() == size); + return target + size; + } else { + io::EpsCopyOutputStream out( + target, size, + io::CodedOutputStream::IsDefaultSerializationDeterministic()); + auto res = msg._InternalSerialize(target, &out); + GOOGLE_DCHECK(target + size == res); + return res; + } +} + uint8* MessageLite::SerializeWithCachedSizesToArray(uint8* target) const { - return InternalSerializeWithCachedSizesToArray( - io::CodedOutputStream::IsDefaultSerializationDeterministic(), target); + // We only optimize this when using optimize_for = SPEED. In other cases + // we just use the CodedOutputStream path. + return SerializeToArrayImpl(*this, target, GetCachedSize()); } bool MessageLite::SerializeToCodedStream(io::CodedOutputStream* output) const { @@ -236,76 +377,101 @@ bool MessageLite::SerializePartialToCodedStream( io::CodedOutputStream* output) const { const size_t size = ByteSizeLong(); // Force size to be cached. if (size > INT_MAX) { - GOOGLE_LOG(ERROR) << "Exceeded maximum protobuf size of 2GB: " << size; + GOOGLE_LOG(ERROR) << GetTypeName() + << " exceeded maximum protobuf size of 2GB: " << size; return false; } - uint8* buffer = output->GetDirectBufferForNBytesAndAdvance(size); - if (buffer != NULL) { - uint8* end = InternalSerializeWithCachedSizesToArray( - output->IsSerializationDeterministic(), buffer); - if (end - buffer != size) { - ByteSizeConsistencyError(size, ByteSizeLong(), end - buffer, *this); - } - return true; - } else { - int original_byte_count = output->ByteCount(); - SerializeWithCachedSizes(output); - if (output->HadError()) { - return false; - } - int final_byte_count = output->ByteCount(); - - if (final_byte_count - original_byte_count != size) { - ByteSizeConsistencyError(size, ByteSizeLong(), - final_byte_count - original_byte_count, *this); - } + int original_byte_count = output->ByteCount(); + SerializeWithCachedSizes(output); + if (output->HadError()) { + return false; + } + int final_byte_count = output->ByteCount(); - return true; + if (final_byte_count - original_byte_count != size) { + ByteSizeConsistencyError(size, ByteSizeLong(), + final_byte_count - original_byte_count, *this); } + + return true; } bool MessageLite::SerializeToZeroCopyStream( io::ZeroCopyOutputStream* output) const { - io::CodedOutputStream encoder(output); - return SerializeToCodedStream(&encoder); + GOOGLE_DCHECK(IsInitialized()) << InitializationErrorMessage("serialize", *this); + return SerializePartialToZeroCopyStream(output); } bool MessageLite::SerializePartialToZeroCopyStream( io::ZeroCopyOutputStream* output) const { - io::CodedOutputStream encoder(output); - return SerializePartialToCodedStream(&encoder); + const size_t size = ByteSizeLong(); // Force size to be cached. + if (size > INT_MAX) { + GOOGLE_LOG(ERROR) << GetTypeName() + << " exceeded maximum protobuf size of 2GB: " << size; + return false; + } + + uint8* target; + io::EpsCopyOutputStream stream( + output, io::CodedOutputStream::IsDefaultSerializationDeterministic(), + &target); + target = _InternalSerialize(target, &stream); + stream.Trim(target); + if (stream.HadError()) return false; + return true; } -bool MessageLite::AppendToString(string* output) const { +bool MessageLite::SerializeToFileDescriptor(int file_descriptor) const { + io::FileOutputStream output(file_descriptor); + return SerializeToZeroCopyStream(&output) && output.Flush(); +} + +bool MessageLite::SerializePartialToFileDescriptor(int file_descriptor) const { + io::FileOutputStream output(file_descriptor); + return SerializePartialToZeroCopyStream(&output) && output.Flush(); +} + +bool MessageLite::SerializeToOstream(std::ostream* output) const { + { + io::OstreamOutputStream zero_copy_output(output); + if (!SerializeToZeroCopyStream(&zero_copy_output)) return false; + } + return output->good(); +} + +bool MessageLite::SerializePartialToOstream(std::ostream* output) const { + io::OstreamOutputStream zero_copy_output(output); + return SerializePartialToZeroCopyStream(&zero_copy_output); +} + +bool MessageLite::AppendToString(std::string* output) const { GOOGLE_DCHECK(IsInitialized()) << InitializationErrorMessage("serialize", *this); return AppendPartialToString(output); } -bool MessageLite::AppendPartialToString(string* output) const { +bool MessageLite::AppendPartialToString(std::string* output) const { size_t old_size = output->size(); size_t byte_size = ByteSizeLong(); if (byte_size > INT_MAX) { - GOOGLE_LOG(ERROR) << "Exceeded maximum protobuf size of 2GB: " << byte_size; + GOOGLE_LOG(ERROR) << GetTypeName() + << " exceeded maximum protobuf size of 2GB: " << byte_size; return false; } STLStringResizeUninitialized(output, old_size + byte_size); uint8* start = reinterpret_cast(io::mutable_string_data(output) + old_size); - uint8* end = SerializeWithCachedSizesToArray(start); - if (end - start != byte_size) { - ByteSizeConsistencyError(byte_size, ByteSizeLong(), end - start, *this); - } + SerializeToArrayImpl(*this, start, byte_size); return true; } -bool MessageLite::SerializeToString(string* output) const { +bool MessageLite::SerializeToString(std::string* output) const { output->clear(); return AppendToString(output); } -bool MessageLite::SerializePartialToString(string* output) const { +bool MessageLite::SerializePartialToString(std::string* output) const { output->clear(); return AppendPartialToString(output); } @@ -316,71 +482,40 @@ bool MessageLite::SerializeToArray(void* data, int size) const { } bool MessageLite::SerializePartialToArray(void* data, int size) const { - int byte_size = ByteSizeLong(); + const size_t byte_size = ByteSizeLong(); + if (byte_size > INT_MAX) { + GOOGLE_LOG(ERROR) << GetTypeName() + << " exceeded maximum protobuf size of 2GB: " << byte_size; + return false; + } if (size < byte_size) return false; uint8* start = reinterpret_cast(data); - uint8* end = SerializeWithCachedSizesToArray(start); - if (end - start != byte_size) { - ByteSizeConsistencyError(byte_size, ByteSizeLong(), end - start, *this); - } + SerializeToArrayImpl(*this, start, byte_size); return true; } -string MessageLite::SerializeAsString() const { +std::string MessageLite::SerializeAsString() const { // If the compiler implements the (Named) Return Value Optimization, // the local variable 'output' will not actually reside on the stack // of this function, but will be overlaid with the object that the // caller supplied for the return value to be constructed in. - string output; - if (!AppendToString(&output)) - output.clear(); + std::string output; + if (!AppendToString(&output)) output.clear(); return output; } -string MessageLite::SerializePartialAsString() const { - string output; - if (!AppendPartialToString(&output)) - output.clear(); +std::string MessageLite::SerializePartialAsString() const { + std::string output; + if (!AppendPartialToString(&output)) output.clear(); return output; } -void MessageLite::SerializeWithCachedSizes( - io::CodedOutputStream* output) const { - GOOGLE_DCHECK(InternalGetTable()); - internal::TableSerialize( - *this, - static_cast(InternalGetTable()), - output); -} - -// The table driven code optimizes the case that the CodedOutputStream buffer -// is large enough to serialize into it directly. -// If the proto is optimized for speed, this method will be overridden by -// generated code for maximum speed. If the proto is optimized for size or -// is lite, then we need to specialize this to avoid infinite recursion. -uint8* MessageLite::InternalSerializeWithCachedSizesToArray( - bool deterministic, uint8* target) const { - const internal::SerializationTable* table = - static_cast(InternalGetTable()); - if (table == NULL) { - // We only optimize this when using optimize_for = SPEED. In other cases - // we just use the CodedOutputStream path. - int size = GetCachedSize(); - io::ArrayOutputStream out(target, size); - io::CodedOutputStream coded_out(&out); - coded_out.SetSerializationDeterministic(deterministic); - SerializeWithCachedSizes(&coded_out); - GOOGLE_CHECK(!coded_out.HadError()); - return target + size; - } else { - return internal::TableSerializeToArray(*this, table, deterministic, target); - } -} namespace internal { -template<> + +template <> MessageLite* GenericTypeHandler::NewFromPrototype( - const MessageLite* prototype, google::protobuf::Arena* arena) { + const MessageLite* prototype, Arena* arena) { return prototype->New(arena); } template <> @@ -388,20 +523,61 @@ void GenericTypeHandler::Merge(const MessageLite& from, MessageLite* to) { to->CheckTypeAndMergeFrom(from); } -template<> -void GenericTypeHandler::Merge(const string& from, - string* to) { +template <> +void GenericTypeHandler::Merge(const std::string& from, + std::string* to) { *to = from; } -bool proto3_preserve_unknown_ = true; +} // namespace internal + -void SetProto3PreserveUnknownsDefault(bool preserve) { - proto3_preserve_unknown_ = preserve; +// =================================================================== +// Shutdown support. + +namespace internal { + +struct ShutdownData { + ~ShutdownData() { + std::reverse(functions.begin(), functions.end()); + for (auto pair : functions) pair.first(pair.second); + } + + static ShutdownData* get() { + static auto* data = new ShutdownData; + return data; + } + + std::vector> functions; + Mutex mutex; +}; + +static void RunZeroArgFunc(const void* arg) { + void (*func)() = reinterpret_cast(const_cast(arg)); + func(); } +void OnShutdown(void (*func)()) { + OnShutdownRun(RunZeroArgFunc, reinterpret_cast(func)); +} + +void OnShutdownRun(void (*f)(const void*), const void* arg) { + auto shutdown_data = ShutdownData::get(); + MutexLock lock(&shutdown_data->mutex); + shutdown_data->functions.push_back(std::make_pair(f, arg)); +} } // namespace internal +void ShutdownProtobufLibrary() { + // This function should be called only once, but accepts multiple calls. + static bool is_shutdown = false; + if (!is_shutdown) { + delete internal::ShutdownData::get(); + is_shutdown = true; + } +} + + } // namespace protobuf } // namespace google diff --git a/third_party/protobuf-lite/parse_context.cc b/third_party/protobuf-lite/parse_context.cc new file mode 100644 index 00000000..ea4ed196 --- /dev/null +++ b/third_party/protobuf-lite/parse_context.cc @@ -0,0 +1,579 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace google { +namespace protobuf { +namespace internal { + +namespace { + +// Only call if at start of tag. +bool ParseEndsInSlopRegion(const char* begin, int overrun, int d) { + constexpr int kSlopBytes = EpsCopyInputStream::kSlopBytes; + GOOGLE_DCHECK(overrun >= 0); + GOOGLE_DCHECK(overrun <= kSlopBytes); + auto ptr = begin + overrun; + auto end = begin + kSlopBytes; + while (ptr < end) { + uint32 tag; + ptr = ReadTag(ptr, &tag); + if (ptr == nullptr || ptr > end) return false; + // ending on 0 tag is allowed and is the major reason for the necessity of + // this function. + if (tag == 0) return true; + switch (tag & 7) { + case 0: { // Varint + uint64 val; + ptr = VarintParse(ptr, &val); + if (ptr == nullptr) return false; + break; + } + case 1: { // fixed64 + ptr += 8; + break; + } + case 2: { // len delim + int32 size = ReadSize(&ptr); + if (ptr == nullptr || size > end - ptr) return false; + ptr += size; + break; + } + case 3: { // start group + d++; + break; + } + case 4: { // end group + if (--d < 0) return true; // We exit early + break; + } + case 5: { // fixed32 + ptr += 4; + break; + } + default: + return false; // Unknown wireformat + } + } + return false; +} + +} // namespace + +const char* EpsCopyInputStream::Next(int overrun, int d) { + if (next_chunk_ == nullptr) return nullptr; // We've reached end of stream. + if (next_chunk_ != buffer_) { + GOOGLE_DCHECK(size_ > kSlopBytes); + // The chunk is large enough to be used directly + buffer_end_ = next_chunk_ + size_ - kSlopBytes; + auto res = next_chunk_; + next_chunk_ = buffer_; + if (aliasing_ == kOnPatch) aliasing_ = kNoDelta; + return res; + } + // Move the slop bytes of previous buffer to start of the patch buffer. + // Note we must use memmove because the previous buffer could be part of + // buffer_. + std::memmove(buffer_, buffer_end_, kSlopBytes); + if (overall_limit_ > 0 && + (d < 0 || !ParseEndsInSlopRegion(buffer_, overrun, d))) { + const void* data; + // ZeroCopyInputStream indicates Next may return 0 size buffers. Hence + // we loop. + while (StreamNext(&data)) { + if (size_ > kSlopBytes) { + // We got a large chunk + std::memcpy(buffer_ + kSlopBytes, data, kSlopBytes); + next_chunk_ = static_cast(data); + buffer_end_ = buffer_ + kSlopBytes; + if (aliasing_ >= kNoDelta) aliasing_ = kOnPatch; + return buffer_; + } else if (size_ > 0) { + std::memcpy(buffer_ + kSlopBytes, data, size_); + next_chunk_ = buffer_; + buffer_end_ = buffer_ + size_; + if (aliasing_ >= kNoDelta) aliasing_ = kOnPatch; + return buffer_; + } + GOOGLE_DCHECK(size_ == 0) << size_; + } + overall_limit_ = 0; // Next failed, no more needs for next + } + // End of stream or array + if (aliasing_ == kNoDelta) { + // If there is no more block and aliasing is true, the previous block + // is still valid and we can alias. We have users relying on string_view's + // obtained from protos to outlive the proto, when the parse was from an + // array. This guarantees string_view's are always aliased if parsed from + // an array. + aliasing_ = reinterpret_cast(buffer_end_) - + reinterpret_cast(buffer_); + } + next_chunk_ = nullptr; + buffer_end_ = buffer_ + kSlopBytes; + size_ = 0; + return buffer_; +} + +std::pair EpsCopyInputStream::DoneFallback(const char* ptr, + int d) { + GOOGLE_DCHECK(ptr >= limit_end_); + int overrun = ptr - buffer_end_; + GOOGLE_DCHECK(overrun <= kSlopBytes); // Guaranteed by parse loop. + // Did we exceeded the limit (parse error). + if (PROTOBUF_PREDICT_FALSE(overrun > limit_)) return {nullptr, true}; + GOOGLE_DCHECK(overrun != limit_); // Guaranteed by caller. + GOOGLE_DCHECK(overrun < limit_); // Follows from above + // TODO(gerbens) Instead of this dcheck we could just assign, and remove + // updating the limit_end from PopLimit, ie. + // limit_end_ = buffer_end_ + (std::min)(0, limit_); + // if (ptr < limit_end_) return {ptr, false}; + GOOGLE_DCHECK(limit_end_ == buffer_end_ + (std::min)(0, limit_)); + // At this point we know the following assertion holds. + GOOGLE_DCHECK(limit_ > 0); + GOOGLE_DCHECK(limit_end_ == buffer_end_); // because limit_ > 0 + do { + // We are past the end of buffer_end_, in the slop region. + GOOGLE_DCHECK(overrun >= 0); + auto p = Next(overrun, d); + if (p == nullptr) { + // We are at the end of the stream + if (PROTOBUF_PREDICT_FALSE(overrun != 0)) return {nullptr, true}; + GOOGLE_DCHECK(limit_ > 0); + limit_end_ = buffer_end_; + // Distinquish ending on a pushed limit or ending on end-of-stream. + SetEndOfStream(); + return {ptr, true}; + } + limit_ -= buffer_end_ - p; // Adjust limit_ relative to new anchor + ptr = p + overrun; + overrun = ptr - buffer_end_; + } while (overrun >= 0); + limit_end_ = buffer_end_ + std::min(0, limit_); + return {ptr, false}; +} + +const char* EpsCopyInputStream::SkipFallback(const char* ptr, int size) { + return AppendSize(ptr, size, [](const char* p, int s) {}); +} + +const char* EpsCopyInputStream::ReadStringFallback(const char* ptr, int size, + std::string* str) { + str->clear(); + if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) { + // Reserve the string up to a static safe size. If strings are bigger than + // this we proceed by growing the string as needed. This protects against + // malicious payloads making protobuf hold on to a lot of memory. + str->reserve(str->size() + std::min(size, kSafeStringSize)); + } + return AppendSize(ptr, size, + [str](const char* p, int s) { str->append(p, s); }); +} + +const char* EpsCopyInputStream::AppendStringFallback(const char* ptr, int size, + std::string* str) { + if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) { + // Reserve the string up to a static safe size. If strings are bigger than + // this we proceed by growing the string as needed. This protects against + // malicious payloads making protobuf hold on to a lot of memory. + str->reserve(str->size() + std::min(size, kSafeStringSize)); + } + return AppendSize(ptr, size, + [str](const char* p, int s) { str->append(p, s); }); +} + + +template +const char* EpsCopyInputStream::ReadRepeatedFixed(const char* ptr, + Tag expected_tag, + RepeatedField* out) { + do { + out->Add(UnalignedLoad(ptr)); + ptr += sizeof(T); + if (PROTOBUF_PREDICT_FALSE(ptr >= limit_end_)) return ptr; + } while (UnalignedLoad(ptr) == expected_tag&& ptr += sizeof(Tag)); + return ptr; +} + +template +void byteswap(void* p); +template <> +void byteswap<1>(void* p) {} +template <> +void byteswap<4>(void* p) { + *static_cast(p) = bswap_32(*static_cast(p)); +} +template <> +void byteswap<8>(void* p) { + *static_cast(p) = bswap_64(*static_cast(p)); +} + +template +const char* EpsCopyInputStream::ReadPackedFixed(const char* ptr, int size, + RepeatedField* out) { + int nbytes = buffer_end_ + kSlopBytes - ptr; + while (size > nbytes) { + int num = nbytes / sizeof(T); + int old_entries = out->size(); + out->Reserve(old_entries + num); + int block_size = num * sizeof(T); + auto dst = out->AddNAlreadyReserved(num); +#ifdef PROTOBUF_LITTLE_ENDIAN + std::memcpy(dst, ptr, block_size); +#else + for (int i = 0; i < num; i++) + dst[i] = UnalignedLoad(ptr + i * sizeof(T)); +#endif + ptr += block_size; + size -= block_size; + if (DoneWithCheck(&ptr, -1)) return nullptr; + nbytes = buffer_end_ + kSlopBytes - ptr; + } + int num = size / sizeof(T); + int old_entries = out->size(); + out->Reserve(old_entries + num); + int block_size = num * sizeof(T); + auto dst = out->AddNAlreadyReserved(num); +#ifdef PROTOBUF_LITTLE_ENDIAN + std::memcpy(dst, ptr, block_size); +#else + for (int i = 0; i < num; i++) dst[i] = UnalignedLoad(ptr + i * sizeof(T)); +#endif + ptr += block_size; + if (size != block_size) return nullptr; + return ptr; +} + +const char* EpsCopyInputStream::InitFrom(io::ZeroCopyInputStream* zcis) { + zcis_ = zcis; + const void* data; + int size; + limit_ = INT_MAX; + if (zcis->Next(&data, &size)) { + overall_limit_ -= size; + if (size > kSlopBytes) { + auto ptr = static_cast(data); + limit_ -= size - kSlopBytes; + limit_end_ = buffer_end_ = ptr + size - kSlopBytes; + next_chunk_ = buffer_; + if (aliasing_ == kOnPatch) aliasing_ = kNoDelta; + return ptr; + } else { + limit_end_ = buffer_end_ = buffer_ + kSlopBytes; + next_chunk_ = buffer_; + auto ptr = buffer_ + 2 * kSlopBytes - size; + std::memcpy(ptr, data, size); + return ptr; + } + } + overall_limit_ = 0; + next_chunk_ = nullptr; + size_ = 0; + limit_end_ = buffer_end_ = buffer_; + return buffer_; +} + +const char* ParseContext::ParseMessage(MessageLite* msg, const char* ptr) { + return ParseMessage(msg, ptr); +} +const char* ParseContext::ParseMessage(Message* msg, const char* ptr) { + // Use reinterptret case to prevent inclusion of non lite header + return ParseMessage(reinterpret_cast(msg), ptr); +} + +inline void WriteVarint(uint64 val, std::string* s) { + while (val >= 128) { + uint8 c = val | 0x80; + s->push_back(c); + val >>= 7; + } + s->push_back(val); +} + +void WriteVarint(uint32 num, uint64 val, std::string* s) { + WriteVarint(num << 3, s); + WriteVarint(val, s); +} + +void WriteLengthDelimited(uint32 num, StringPiece val, std::string* s) { + WriteVarint((num << 3) + 2, s); + WriteVarint(val.size(), s); + s->append(val.data(), val.size()); +} + +std::pair VarintParseSlow32(const char* p, uint32 res) { + for (std::uint32_t i = 2; i < 5; i++) { + uint32 byte = static_cast(p[i]); + res += (byte - 1) << (7 * i); + if (PROTOBUF_PREDICT_TRUE(byte < 128)) { + return {p + i + 1, res}; + } + } + // Accept >5 bytes + for (std::uint32_t i = 5; i < 10; i++) { + uint32 byte = static_cast(p[i]); + if (PROTOBUF_PREDICT_TRUE(byte < 128)) { + return {p + i + 1, res}; + } + } + return {nullptr, 0}; +} + +std::pair VarintParseSlow64(const char* p, uint32 res32) { + uint64 res = res32; + for (std::uint32_t i = 2; i < 10; i++) { + uint64 byte = static_cast(p[i]); + res += (byte - 1) << (7 * i); + if (PROTOBUF_PREDICT_TRUE(byte < 128)) { + return {p + i + 1, res}; + } + } + return {nullptr, 0}; +} + +std::pair ReadTagFallback(const char* p, uint32 res) { + for (std::uint32_t i = 2; i < 5; i++) { + uint32 byte = static_cast(p[i]); + res += (byte - 1) << (7 * i); + if (PROTOBUF_PREDICT_TRUE(byte < 128)) { + return {p + i + 1, res}; + } + } + return {nullptr, 0}; +} + +std::pair ReadSizeFallback(const char* p, uint32 res) { + for (std::uint32_t i = 1; i < 4; i++) { + uint32 byte = static_cast(p[i]); + res += (byte - 1) << (7 * i); + if (PROTOBUF_PREDICT_TRUE(byte < 128)) { + return {p + i + 1, res}; + } + } + std::uint32_t byte = static_cast(p[4]); + if (PROTOBUF_PREDICT_FALSE(byte >= 8)) return {nullptr, 0}; // size >= 2gb + res += (byte - 1) << 28; + // Protect against sign integer overflow in PushLimit. Limits are relative + // to buffer ends and ptr could potential be kSlopBytes beyond a buffer end. + // To protect against overflow we reject limits absurdly close to INT_MAX. + if (PROTOBUF_PREDICT_FALSE(res > INT_MAX - ParseContext::kSlopBytes)) { + return {nullptr, 0}; + } + return {p + 5, res}; +} + +const char* StringParser(const char* begin, const char* end, void* object, + ParseContext*) { + auto str = static_cast(object); + str->append(begin, end - begin); + return end; +} + +// Defined in wire_format_lite.cc +void PrintUTF8ErrorLog(const char* field_name, const char* operation_str, + bool emit_stacktrace); + +bool VerifyUTF8(StringPiece str, const char* field_name) { + if (!IsStructurallyValidUTF8(str)) { + PrintUTF8ErrorLog(field_name, "parsing", false); + return false; + } + return true; +} + +const char* InlineGreedyStringParser(std::string* s, const char* ptr, + ParseContext* ctx) { + int size = ReadSize(&ptr); + if (!ptr) return nullptr; + return ctx->ReadString(ptr, size, s); +} + + +template +const char* VarintParser(void* object, const char* ptr, ParseContext* ctx) { + return ctx->ReadPackedVarint(ptr, [object](uint64 varint) { + T val; + if (sign) { + if (sizeof(T) == 8) { + val = WireFormatLite::ZigZagDecode64(varint); + } else { + val = WireFormatLite::ZigZagDecode32(varint); + } + } else { + val = varint; + } + static_cast*>(object)->Add(val); + }); +} + +const char* PackedInt32Parser(void* object, const char* ptr, + ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} +const char* PackedUInt32Parser(void* object, const char* ptr, + ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} +const char* PackedInt64Parser(void* object, const char* ptr, + ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} +const char* PackedUInt64Parser(void* object, const char* ptr, + ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} +const char* PackedSInt32Parser(void* object, const char* ptr, + ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} +const char* PackedSInt64Parser(void* object, const char* ptr, + ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} + +const char* PackedEnumParser(void* object, const char* ptr, ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} + +const char* PackedBoolParser(void* object, const char* ptr, ParseContext* ctx) { + return VarintParser(object, ptr, ctx); +} + +template +const char* FixedParser(void* object, const char* ptr, ParseContext* ctx) { + int size = ReadSize(&ptr); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + return ctx->ReadPackedFixed(ptr, size, + static_cast*>(object)); +} + +const char* PackedFixed32Parser(void* object, const char* ptr, + ParseContext* ctx) { + return FixedParser(object, ptr, ctx); +} +const char* PackedSFixed32Parser(void* object, const char* ptr, + ParseContext* ctx) { + return FixedParser(object, ptr, ctx); +} +const char* PackedFixed64Parser(void* object, const char* ptr, + ParseContext* ctx) { + return FixedParser(object, ptr, ctx); +} +const char* PackedSFixed64Parser(void* object, const char* ptr, + ParseContext* ctx) { + return FixedParser(object, ptr, ctx); +} +const char* PackedFloatParser(void* object, const char* ptr, + ParseContext* ctx) { + return FixedParser(object, ptr, ctx); +} +const char* PackedDoubleParser(void* object, const char* ptr, + ParseContext* ctx) { + return FixedParser(object, ptr, ctx); +} + +class UnknownFieldLiteParserHelper { + public: + explicit UnknownFieldLiteParserHelper(std::string* unknown) + : unknown_(unknown) {} + + void AddVarint(uint32 num, uint64 value) { + if (unknown_ == nullptr) return; + WriteVarint(num * 8, unknown_); + WriteVarint(value, unknown_); + } + void AddFixed64(uint32 num, uint64 value) { + if (unknown_ == nullptr) return; + WriteVarint(num * 8 + 1, unknown_); + char buffer[8]; + io::CodedOutputStream::WriteLittleEndian64ToArray( + value, reinterpret_cast(buffer)); + unknown_->append(buffer, 8); + } + const char* ParseLengthDelimited(uint32 num, const char* ptr, + ParseContext* ctx) { + int size = ReadSize(&ptr); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + if (unknown_ == nullptr) return ctx->Skip(ptr, size); + WriteVarint(num * 8 + 2, unknown_); + WriteVarint(size, unknown_); + return ctx->AppendString(ptr, size, unknown_); + } + const char* ParseGroup(uint32 num, const char* ptr, ParseContext* ctx) { + if (unknown_) WriteVarint(num * 8 + 3, unknown_); + ptr = ctx->ParseGroup(this, ptr, num * 8 + 3); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + if (unknown_) WriteVarint(num * 8 + 4, unknown_); + return ptr; + } + void AddFixed32(uint32 num, uint32 value) { + if (unknown_ == nullptr) return; + WriteVarint(num * 8 + 5, unknown_); + char buffer[4]; + io::CodedOutputStream::WriteLittleEndian32ToArray( + value, reinterpret_cast(buffer)); + unknown_->append(buffer, 4); + } + + const char* _InternalParse(const char* ptr, ParseContext* ctx) { + return WireFormatParser(*this, ptr, ctx); + } + + private: + std::string* unknown_; +}; + +const char* UnknownGroupLiteParse(std::string* unknown, const char* ptr, + ParseContext* ctx) { + UnknownFieldLiteParserHelper field_parser(unknown); + return WireFormatParser(field_parser, ptr, ctx); +} + +const char* UnknownFieldParse(uint32 tag, std::string* unknown, const char* ptr, + ParseContext* ctx) { + UnknownFieldLiteParserHelper field_parser(unknown); + return FieldParser(tag, field_parser, ptr, ctx); +} + +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/third_party/protobuf-lite/repeated_field.cc b/third_party/protobuf-lite/repeated_field.cc index 310000aa..64506791 100644 --- a/third_party/protobuf-lite/repeated_field.cc +++ b/third_party/protobuf-lite/repeated_field.cc @@ -32,11 +32,15 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. +#include + #include -#include #include #include +#include + +#include namespace google { namespace protobuf { @@ -51,19 +55,17 @@ void** RepeatedPtrFieldBase::InternalExtend(int extend_amount) { return &rep_->elements[current_size_]; } Rep* old_rep = rep_; - Arena* arena = GetArenaNoVirtual(); - new_size = std::max(kMinRepeatedFieldAllocationSize, + Arena* arena = GetArena(); + new_size = std::max(internal::kRepeatedFieldLowerClampLimit, std::max(total_size_ * 2, new_size)); - GOOGLE_CHECK_LE(new_size, - (std::numeric_limits::max() - kRepHeaderSize) / - sizeof(old_rep->elements[0])) + GOOGLE_CHECK_LE(new_size, (std::numeric_limits::max() - kRepHeaderSize) / + sizeof(old_rep->elements[0])) << "Requested size is too large to fit into size_t."; size_t bytes = kRepHeaderSize + sizeof(old_rep->elements[0]) * new_size; if (arena == NULL) { rep_ = reinterpret_cast(::operator new(bytes)); } else { - rep_ = reinterpret_cast( - ::google::protobuf::Arena::CreateArray(arena, bytes)); + rep_ = reinterpret_cast(Arena::CreateArray(arena, bytes)); } #if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) const int old_total_size = total_size_; @@ -103,18 +105,17 @@ void RepeatedPtrFieldBase::CloseGap(int start, int num) { rep_->allocated_size -= num; } -google::protobuf::MessageLite* RepeatedPtrFieldBase::AddWeak( - const google::protobuf::MessageLite* prototype) { +MessageLite* RepeatedPtrFieldBase::AddWeak(const MessageLite* prototype) { if (rep_ != NULL && current_size_ < rep_->allocated_size) { - return reinterpret_cast( - rep_->elements[current_size_++]); + return reinterpret_cast(rep_->elements[current_size_++]); } if (!rep_ || rep_->allocated_size == total_size_) { Reserve(total_size_ + 1); } ++rep_->allocated_size; - google::protobuf::MessageLite* result = prototype ? prototype->New(arena_) : - Arena::CreateMessage(arena_); + MessageLite* result = prototype + ? prototype->New(arena_) + : Arena::CreateMessage(arena_); rep_->elements[current_size_++] = result; return result; } @@ -122,5 +123,14 @@ google::protobuf::MessageLite* RepeatedPtrFieldBase::AddWeak( } // namespace internal +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField; +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField; +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField; +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField; +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField; +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField; +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField; +template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedPtrField; + } // namespace protobuf } // namespace google diff --git a/third_party/protobuf-lite/statusor.cc b/third_party/protobuf-lite/statusor.cc index 48d1402a..c744b8d2 100644 --- a/third_party/protobuf-lite/statusor.cc +++ b/third_party/protobuf-lite/statusor.cc @@ -30,6 +30,8 @@ #include +#include + namespace google { namespace protobuf { namespace util { diff --git a/third_party/protobuf-lite/stringpiece.cc b/third_party/protobuf-lite/stringpiece.cc index 989474b7..fc2f5210 100644 --- a/third_party/protobuf-lite/stringpiece.cc +++ b/third_party/protobuf-lite/stringpiece.cc @@ -35,6 +35,8 @@ #include #include +#include + namespace google { namespace protobuf { std::ostream& operator<<(std::ostream& o, StringPiece piece) { @@ -115,7 +117,7 @@ stringpiece_ssize_type StringPiece::find(char c, size_type pos) const { } const char* result = static_cast( memchr(ptr_ + pos, c, length_ - pos)); - return result != NULL ? result - ptr_ : npos; + return result != nullptr ? result - ptr_ : npos; } stringpiece_ssize_type StringPiece::rfind(StringPiece s, size_type pos) const { diff --git a/third_party/protobuf-lite/stringprintf.cc b/third_party/protobuf-lite/stringprintf.cc index d98b9b87..e6019fc6 100644 --- a/third_party/protobuf-lite/stringprintf.cc +++ b/third_party/protobuf-lite/stringprintf.cc @@ -36,7 +36,9 @@ #include // For va_list and related operations #include // MSVC requires this for _vsnprintf #include + #include +#include namespace google { namespace protobuf { @@ -76,7 +78,7 @@ void StringAppendV(string* dst, const char* format, va_list ap) { // Error or MSVC running out of space. MSVC 8.0 and higher // can be asked about space needed with the special idiom below: va_copy(backup_ap, ap); - result = vsnprintf(NULL, 0, format, backup_ap); + result = vsnprintf(nullptr, 0, format, backup_ap); va_end(backup_ap); } diff --git a/third_party/protobuf-lite/structurally_valid.cc b/third_party/protobuf-lite/structurally_valid.cc index b2239682..4d424a15 100644 --- a/third_party/protobuf-lite/structurally_valid.cc +++ b/third_party/protobuf-lite/structurally_valid.cc @@ -395,7 +395,7 @@ int UTF8GenericScan(const UTF8ScanObj* st, const uint8* isrc = reinterpret_cast(str); const uint8* src = isrc; const uint8* srclimit = isrc + str_length; - const uint8* srclimit8 = srclimit - 7; + const uint8* srclimit8 = str_length < 7 ? isrc : srclimit - 7; const uint8* Tbl_0 = &st->state_table[st->state0]; DoAgain: @@ -456,8 +456,7 @@ int UTF8GenericScan(const UTF8ScanObj* st, } //---------------------------- - - // Exit posibilities: + // Exit possibilities: // Some exit code, !state0, back up over last char // Some exit code, state0, back up one byte exactly // source consumed, !state0, back up over partial char @@ -504,7 +503,7 @@ int UTF8GenericScanFastAscii(const UTF8ScanObj* st, const uint8* isrc = reinterpret_cast(str); const uint8* src = isrc; const uint8* srclimit = isrc + str_length; - const uint8* srclimit8 = srclimit - 7; + const uint8* srclimit8 = str_length < 7 ? isrc : srclimit - 7; int n; int rest_consumed; int exit_reason; @@ -555,7 +554,7 @@ InitDetector init_detector; bool IsStructurallyValidUTF8(const char* buf, int len) { if (!module_initialized_) return true; - + int bytes_consumed = 0; UTF8GenericScanFastAscii(&utf8acceptnonsurrogates_obj, buf, len, &bytes_consumed); diff --git a/third_party/protobuf-lite/strutil.cc b/third_party/protobuf-lite/strutil.cc index 552d416f..24ae286d 100644 --- a/third_party/protobuf-lite/strutil.cc +++ b/third_party/protobuf-lite/strutil.cc @@ -31,15 +31,16 @@ // from google3/strings/strutil.cc #include -#include #include #include // FLT_DIG and DBL_DIG -#include #include #include +#include #include +#include +#include #include #ifdef _WIN32 @@ -78,21 +79,6 @@ inline bool isprint(char c) { return c >= 0x20 && c <= 0x7E; } -// ---------------------------------------------------------------------- -// StripString -// Replaces any occurrence of the character 'remove' (or the characters -// in 'remove') with the character 'replacewith'. -// ---------------------------------------------------------------------- -void StripString(string* s, const char* remove, char replacewith) { - const char * str_start = s->c_str(); - const char * str = str_start; - for (str = strpbrk(str, remove); - str != NULL; - str = strpbrk(str + 1, remove)) { - (*s)[str - str_start] = replacewith; - } -} - // ---------------------------------------------------------------------- // ReplaceCharacters // Replaces any occurrence of the character 'remove' (or the characters @@ -102,7 +88,7 @@ void ReplaceCharacters(string *s, const char *remove, char replacewith) { const char *str_start = s->c_str(); const char *str = str_start; for (str = strpbrk(str, remove); - str != NULL; + str != nullptr; str = strpbrk(str + 1, remove)) { (*s)[str - str_start] = replacewith; } @@ -280,7 +266,7 @@ static void JoinStringsIterator(const ITERATOR& start, const ITERATOR& end, const char* delim, string* result) { - GOOGLE_CHECK(result != NULL); + GOOGLE_CHECK(result != nullptr); result->clear(); int delim_length = strlen(delim); @@ -318,7 +304,7 @@ void JoinStrings(const std::vector& components, // result is truncated to 8 bits. // // The second call stores its errors in a supplied string vector. -// If the string vector pointer is NULL, it reports the errors with LOG(). +// If the string vector pointer is nullptr, it reports the errors with LOG(). // ---------------------------------------------------------------------- #define IS_OCTAL_DIGIT(c) (((c) >= '0') && ((c) <= '7')) @@ -328,12 +314,12 @@ void JoinStrings(const std::vector& components, #define LOG_STRING(LEVEL, VECTOR) GOOGLE_LOG_IF(LEVEL, false) int UnescapeCEscapeSequences(const char* source, char* dest) { - return UnescapeCEscapeSequences(source, dest, NULL); + return UnescapeCEscapeSequences(source, dest, nullptr); } int UnescapeCEscapeSequences(const char* source, char* dest, std::vector *errors) { - GOOGLE_DCHECK(errors == NULL) << "Error reporting not implemented."; + GOOGLE_DCHECK(errors == nullptr) << "Error reporting not implemented."; char* d = dest; const char* p = source; @@ -458,13 +444,13 @@ int UnescapeCEscapeSequences(const char* source, char* dest, // to be the same. // // The second call stores its errors in a supplied string vector. -// If the string vector pointer is NULL, it reports the errors with LOG(). +// If the string vector pointer is nullptr, it reports the errors with LOG(). // // In the first and second calls, the length of dest is returned. In the // the third call, the new string is returned. // ---------------------------------------------------------------------- int UnescapeCEscapeString(const string& src, string* dest) { - return UnescapeCEscapeString(src, dest, NULL); + return UnescapeCEscapeString(src, dest, nullptr); } int UnescapeCEscapeString(const string& src, string* dest, @@ -478,7 +464,7 @@ int UnescapeCEscapeString(const string& src, string* dest, string UnescapeCEscapeString(const string& src) { std::unique_ptr unescaped(new char[src.size() + 1]); - int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), NULL); + int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), nullptr); return string(unescaped.get(), len); } @@ -982,7 +968,7 @@ static const char two_ASCII_digits[100][2] = { char* FastUInt32ToBufferLeft(uint32 u, char* buffer) { uint32 digits; - const char *ASCII_digits = NULL; + const char *ASCII_digits = nullptr; // The idea of this implementation is to trim the number of divides to as few // as possible by using multiplication and subtraction rather than mod (%), // and by outputting two digits at a time rather than one. @@ -1063,17 +1049,19 @@ char* FastUInt32ToBufferLeft(uint32 u, char* buffer) { } char* FastInt32ToBufferLeft(int32 i, char* buffer) { - uint32 u = i; + uint32 u = 0; if (i < 0) { *buffer++ = '-'; - u = -i; + u -= i; + } else { + u = i; } return FastUInt32ToBufferLeft(u, buffer); } char* FastUInt64ToBufferLeft(uint64 u64, char* buffer) { int digits; - const char *ASCII_digits = NULL; + const char *ASCII_digits = nullptr; uint32 u = static_cast(u64); if (u == u64) return FastUInt32ToBufferLeft(u, buffer); @@ -1114,10 +1102,12 @@ char* FastUInt64ToBufferLeft(uint64 u64, char* buffer) { } char* FastInt64ToBufferLeft(int64 i, char* buffer) { - uint64 u = i; + uint64 u = 0; if (i < 0) { *buffer++ = '-'; - u = -i; + u -= i; + } else { + u = i; } return FastUInt64ToBufferLeft(u, buffer); } @@ -1231,7 +1221,7 @@ static inline bool IsValidFloatChar(char c) { void DelocalizeRadix(char* buffer) { // Fast check: if the buffer has a normal decimal point, assume no // translation is needed. - if (strchr(buffer, '.') != NULL) return; + if (strchr(buffer, '.') != nullptr) return; // Find the first unknown character. while (IsValidFloatChar(*buffer)) ++buffer; @@ -1268,7 +1258,7 @@ char* DoubleToBuffer(double value, char* buffer) { } else if (value == -std::numeric_limits::infinity()) { strcpy(buffer, "-inf"); return buffer; - } else if (MathLimits::IsNaN(value)) { + } else if (std::isnan(value)) { strcpy(buffer, "nan"); return buffer; } @@ -1286,7 +1276,7 @@ char* DoubleToBuffer(double value, char* buffer) { // of a double. This long double may have extra bits that make it compare // unequal to "value" even though it would be exactly equal if it were // truncated to a double. - volatile double parsed_value = strtod(buffer, NULL); + volatile double parsed_value = internal::NoLocaleStrtod(buffer, nullptr); if (parsed_value != value) { int snprintf_result = snprintf(buffer, kDoubleToBufferSize, "%.*g", DBL_DIG+2, value); @@ -1318,7 +1308,7 @@ inline bool CaseEqual(StringPiece s1, StringPiece s2) { } bool safe_strtob(StringPiece str, bool* value) { - GOOGLE_CHECK(value != NULL) << "NULL output boolean given."; + GOOGLE_CHECK(value != nullptr) << "nullptr output boolean given."; if (CaseEqual(str, "true") || CaseEqual(str, "t") || CaseEqual(str, "yes") || CaseEqual(str, "y") || CaseEqual(str, "1")) { @@ -1338,7 +1328,7 @@ bool safe_strtof(const char* str, float* value) { char* endptr; errno = 0; // errno only gets set on errors #if defined(_WIN32) || defined (__hpux) // has no strtof() - *value = strtod(str, &endptr); + *value = internal::NoLocaleStrtod(str, &endptr); #else *value = strtof(str, &endptr); #endif @@ -1347,7 +1337,7 @@ bool safe_strtof(const char* str, float* value) { bool safe_strtod(const char* str, double* value) { char* endptr; - *value = strtod(str, &endptr); + *value = internal::NoLocaleStrtod(str, &endptr); if (endptr != str) { while (ascii_isspace(*endptr)) ++endptr; } @@ -1386,7 +1376,7 @@ char* FloatToBuffer(float value, char* buffer) { } else if (value == -std::numeric_limits::infinity()) { strcpy(buffer, "-inf"); return buffer; - } else if (MathLimits::IsNaN(value)) { + } else if (std::isnan(value)) { strcpy(buffer, "nan"); return buffer; } @@ -1445,32 +1435,44 @@ AlphaNum::AlphaNum(strings::Hex hex) { // after the area just overwritten. It comes in multiple flavors to minimize // call overhead. static char *Append1(char *out, const AlphaNum &x) { - memcpy(out, x.data(), x.size()); - return out + x.size(); + if (x.size() > 0) { + memcpy(out, x.data(), x.size()); + out += x.size(); + } + return out; } static char *Append2(char *out, const AlphaNum &x1, const AlphaNum &x2) { - memcpy(out, x1.data(), x1.size()); - out += x1.size(); - - memcpy(out, x2.data(), x2.size()); - return out + x2.size(); + if (x1.size() > 0) { + memcpy(out, x1.data(), x1.size()); + out += x1.size(); + } + if (x2.size() > 0) { + memcpy(out, x2.data(), x2.size()); + out += x2.size(); + } + return out; } -static char *Append4(char *out, - const AlphaNum &x1, const AlphaNum &x2, +static char *Append4(char *out, const AlphaNum &x1, const AlphaNum &x2, const AlphaNum &x3, const AlphaNum &x4) { - memcpy(out, x1.data(), x1.size()); - out += x1.size(); - - memcpy(out, x2.data(), x2.size()); - out += x2.size(); - - memcpy(out, x3.data(), x3.size()); - out += x3.size(); - - memcpy(out, x4.data(), x4.size()); - return out + x4.size(); + if (x1.size() > 0) { + memcpy(out, x1.data(), x1.size()); + out += x1.size(); + } + if (x2.size() > 0) { + memcpy(out, x2.data(), x2.size()); + out += x2.size(); + } + if (x3.size() > 0) { + memcpy(out, x3.data(), x3.size()); + out += x3.size(); + } + if (x4.size() > 0) { + memcpy(out, x4.data(), x4.size()); + out += x4.size(); + } + return out; } string StrCat(const AlphaNum &a, const AlphaNum &b) { @@ -1619,7 +1621,7 @@ void StrAppend(string *result, int GlobalReplaceSubstring(const string& substring, const string& replacement, string* s) { - GOOGLE_CHECK(s != NULL); + GOOGLE_CHECK(s != nullptr); if (s->empty() || substring.empty()) return 0; string tmp; @@ -1959,24 +1961,25 @@ int Base64UnescapeInternal(const char *src_param, int szsrc, // #include // #include // #include +// #include // main() // { // static const char Base64[] = // "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -// char *pos; +// const char *pos; // int idx, i, j; // printf(" "); // for (i = 0; i < 255; i += 8) { // for (j = i; j < i + 8; j++) { // pos = strchr(Base64, j); -// if ((pos == NULL) || (j == 0)) +// if ((pos == nullptr) || (j == 0)) // idx = -1; // else // idx = pos - Base64; // if (idx == -1) // printf(" %2d, ", idx); // else -// printf(" %2d/*%c*/,", idx, j); +// printf(" %2d/""*%c*""/,", idx, j); // } // printf("\n "); // } @@ -1994,7 +1997,7 @@ static const signed char kUnBase64[] = { 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, - 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 7/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, -1, -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, @@ -2028,7 +2031,7 @@ static const signed char kUnWebSafeBase64[] = { 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, - 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 7/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, 63/*_*/, -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, @@ -2281,16 +2284,19 @@ int EncodeAsUTF8Char(uint32 code_point, char* output) { // Table of UTF-8 character lengths, based on first byte static const unsigned char kUTF8LenTbl[256] = { - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, - 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4 -}; + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; // Return length of a single UTF-8 source character int UTF8FirstLetterNumBytes(const char* src, int len) { @@ -2300,5 +2306,181 @@ int UTF8FirstLetterNumBytes(const char* src, int len) { return kUTF8LenTbl[*reinterpret_cast(src)]; } +// ---------------------------------------------------------------------- +// CleanStringLineEndings() +// Clean up a multi-line string to conform to Unix line endings. +// Reads from src and appends to dst, so usually dst should be empty. +// +// If there is no line ending at the end of a non-empty string, it can +// be added automatically. +// +// Four different types of input are correctly handled: +// +// - Unix/Linux files: line ending is LF: pass through unchanged +// +// - DOS/Windows files: line ending is CRLF: convert to LF +// +// - Legacy Mac files: line ending is CR: convert to LF +// +// - Garbled files: random line endings: convert gracefully +// lonely CR, lonely LF, CRLF: convert to LF +// +// @param src The multi-line string to convert +// @param dst The converted string is appended to this string +// @param auto_end_last_line Automatically terminate the last line +// +// Limitations: +// +// This does not do the right thing for CRCRLF files created by +// broken programs that do another Unix->DOS conversion on files +// that are already in CRLF format. For this, a two-pass approach +// brute-force would be needed that +// +// (1) determines the presence of LF (first one is ok) +// (2) if yes, removes any CR, else convert every CR to LF + +void CleanStringLineEndings(const string &src, string *dst, + bool auto_end_last_line) { + if (dst->empty()) { + dst->append(src); + CleanStringLineEndings(dst, auto_end_last_line); + } else { + string tmp = src; + CleanStringLineEndings(&tmp, auto_end_last_line); + dst->append(tmp); + } +} + +void CleanStringLineEndings(string *str, bool auto_end_last_line) { + ptrdiff_t output_pos = 0; + bool r_seen = false; + ptrdiff_t len = str->size(); + + char *p = &(*str)[0]; + + for (ptrdiff_t input_pos = 0; input_pos < len;) { + if (!r_seen && input_pos + 8 < len) { + uint64_t v = GOOGLE_UNALIGNED_LOAD64(p + input_pos); + // Loop over groups of 8 bytes at a time until we come across + // a word that has a byte whose value is less than or equal to + // '\r' (i.e. could contain a \n (0x0a) or a \r (0x0d) ). + // + // We use a has_less macro that quickly tests a whole 64-bit + // word to see if any of the bytes has a value < N. + // + // For more details, see: + // http://graphics.stanford.edu/~seander/bithacks.html#HasLessInWord +#define has_less(x, n) (((x) - ~0ULL / 255 * (n)) & ~(x) & ~0ULL / 255 * 128) + if (!has_less(v, '\r' + 1)) { +#undef has_less + // No byte in this word has a value that could be a \r or a \n + if (output_pos != input_pos) { + GOOGLE_UNALIGNED_STORE64(p + output_pos, v); + } + input_pos += 8; + output_pos += 8; + continue; + } + } + string::const_reference in = p[input_pos]; + if (in == '\r') { + if (r_seen) p[output_pos++] = '\n'; + r_seen = true; + } else if (in == '\n') { + if (input_pos != output_pos) + p[output_pos++] = '\n'; + else + output_pos++; + r_seen = false; + } else { + if (r_seen) p[output_pos++] = '\n'; + r_seen = false; + if (input_pos != output_pos) + p[output_pos++] = in; + else + output_pos++; + } + input_pos++; + } + if (r_seen || + (auto_end_last_line && output_pos > 0 && p[output_pos - 1] != '\n')) { + str->resize(output_pos + 1); + str->operator[](output_pos) = '\n'; + } else if (output_pos < len) { + str->resize(output_pos); + } +} + +namespace internal { + +// ---------------------------------------------------------------------- +// NoLocaleStrtod() +// This code will make you cry. +// ---------------------------------------------------------------------- + +namespace { + +// Returns a string identical to *input except that the character pointed to +// by radix_pos (which should be '.') is replaced with the locale-specific +// radix character. +std::string LocalizeRadix(const char *input, const char *radix_pos) { + // Determine the locale-specific radix character by calling sprintf() to + // print the number 1.5, then stripping off the digits. As far as I can + // tell, this is the only portable, thread-safe way to get the C library + // to divuldge the locale's radix character. No, localeconv() is NOT + // thread-safe. + char temp[16]; + int size = snprintf(temp, sizeof(temp), "%.1f", 1.5); + GOOGLE_CHECK_EQ(temp[0], '1'); + GOOGLE_CHECK_EQ(temp[size - 1], '5'); + GOOGLE_CHECK_LE(size, 6); + + // Now replace the '.' in the input with it. + std::string result; + result.reserve(strlen(input) + size - 3); + result.append(input, radix_pos); + result.append(temp + 1, size - 2); + result.append(radix_pos + 1); + return result; +} + +} // namespace + +double NoLocaleStrtod(const char *str, char **endptr) { + // We cannot simply set the locale to "C" temporarily with setlocale() + // as this is not thread-safe. Instead, we try to parse in the current + // locale first. If parsing stops at a '.' character, then this is a + // pretty good hint that we're actually in some other locale in which + // '.' is not the radix character. + + char *temp_endptr; + double result = strtod(str, &temp_endptr); + if (endptr != NULL) *endptr = temp_endptr; + if (*temp_endptr != '.') return result; + + // Parsing halted on a '.'. Perhaps we're in a different locale? Let's + // try to replace the '.' with a locale-specific radix character and + // try again. + std::string localized = LocalizeRadix(str, temp_endptr); + const char *localized_cstr = localized.c_str(); + char *localized_endptr; + result = strtod(localized_cstr, &localized_endptr); + if ((localized_endptr - localized_cstr) > (temp_endptr - str)) { + // This attempt got further, so replacing the decimal must have helped. + // Update endptr to point at the right location. + if (endptr != NULL) { + // size_diff is non-zero if the localized radix has multiple bytes. + int size_diff = localized.size() - strlen(str); + // const_cast is necessary to match the strtod() interface. + *endptr = const_cast( + str + (localized_endptr - localized_cstr - size_diff)); + } + } + + return result; +} + +} // namespace internal + } // namespace protobuf } // namespace google diff --git a/third_party/protobuf-lite/time.cc b/third_party/protobuf-lite/time.cc index 6def637e..64f3ceb9 100644 --- a/third_party/protobuf-lite/time.cc +++ b/third_party/protobuf-lite/time.cc @@ -142,12 +142,12 @@ string FormatNanos(int32 nanos) { // Parses an integer from a null-terminated char sequence. The method // consumes at most "width" chars. Returns a pointer after the consumed -// integer, or NULL if the data does not start with an integer or the +// integer, or nullptr if the data does not start with an integer or the // integer value does not fall in the range of [min_value, max_value]. const char* ParseInt(const char* data, int width, int min_value, int max_value, int* result) { if (!ascii_isdigit(*data)) { - return NULL; + return nullptr; } int value = 0; for (int i = 0; i < width; ++i, ++data) { @@ -161,7 +161,7 @@ const char* ParseInt(const char* data, int width, int min_value, *result = value; return data; } else { - return NULL; + return nullptr; } } @@ -169,7 +169,7 @@ const char* ParseInt(const char* data, int width, int min_value, // "010" will be parsed to 10000000 nanos. const char* ParseNanos(const char* data, int32* nanos) { if (!ascii_isdigit(*data)) { - return NULL; + return nullptr; } int value = 0; int len = 0; @@ -193,15 +193,15 @@ const char* ParseNanos(const char* data, int32* nanos) { const char* ParseTimezoneOffset(const char* data, int64* offset) { // Accept format "HH:MM". E.g., "08:00" int hour; - if ((data = ParseInt(data, 2, 0, 23, &hour)) == NULL) { - return NULL; + if ((data = ParseInt(data, 2, 0, 23, &hour)) == nullptr) { + return nullptr; } if (*data++ != ':') { - return NULL; + return nullptr; } int minute; - if ((data = ParseInt(data, 2, 0, 59, &minute)) == NULL) { - return NULL; + if ((data = ParseInt(data, 2, 0, 59, &minute)) == nullptr) { + return nullptr; } *offset = (hour * 60 + minute) * 60; return data; @@ -212,7 +212,7 @@ bool SecondsToDateTime(int64 seconds, DateTime* time) { if (seconds < kMinTime || seconds > kMaxTime) { return false; } - // It's easier to calcuate the DateTime starting from 0001-01-01T00:00:00 + // It's easier to calculate the DateTime starting from 0001-01-01T00:00:00 seconds = seconds + kSecondsFromEraToEpoch; int year = 1; if (seconds >= kSecondsPer400Years) { @@ -264,7 +264,7 @@ bool DateTimeToSeconds(const DateTime& time, int64* seconds) { void GetCurrentTime(int64* seconds, int32* nanos) { // TODO(xiaofeng): Improve the accuracy of this implementation (or just // remove this method from protobuf). - *seconds = time(NULL); + *seconds = time(nullptr); *nanos = 0; } @@ -290,37 +290,37 @@ bool ParseTime(const string& value, int64* seconds, int32* nanos) { // With UTC offset: 2015-05-20T13:29:35.120-08:00 // Parse year - if ((data = ParseInt(data, 4, 1, 9999, &time.year)) == NULL) { + if ((data = ParseInt(data, 4, 1, 9999, &time.year)) == nullptr) { return false; } // Expect '-' if (*data++ != '-') return false; // Parse month - if ((data = ParseInt(data, 2, 1, 12, &time.month)) == NULL) { + if ((data = ParseInt(data, 2, 1, 12, &time.month)) == nullptr) { return false; } // Expect '-' if (*data++ != '-') return false; // Parse day - if ((data = ParseInt(data, 2, 1, 31, &time.day)) == NULL) { + if ((data = ParseInt(data, 2, 1, 31, &time.day)) == nullptr) { return false; } // Expect 'T' if (*data++ != 'T') return false; // Parse hour - if ((data = ParseInt(data, 2, 0, 23, &time.hour)) == NULL) { + if ((data = ParseInt(data, 2, 0, 23, &time.hour)) == nullptr) { return false; } // Expect ':' if (*data++ != ':') return false; // Parse minute - if ((data = ParseInt(data, 2, 0, 59, &time.minute)) == NULL) { + if ((data = ParseInt(data, 2, 0, 59, &time.minute)) == nullptr) { return false; } // Expect ':' if (*data++ != ':') return false; // Parse second - if ((data = ParseInt(data, 2, 0, 59, &time.second)) == NULL) { + if ((data = ParseInt(data, 2, 0, 59, &time.second)) == nullptr) { return false; } if (!DateTimeToSeconds(time, seconds)) { @@ -330,7 +330,7 @@ bool ParseTime(const string& value, int64* seconds, int32* nanos) { if (*data == '.') { ++data; // Parse nanoseconds. - if ((data = ParseNanos(data, nanos)) == NULL) { + if ((data = ParseNanos(data, nanos)) == nullptr) { return false; } } else { @@ -342,14 +342,14 @@ bool ParseTime(const string& value, int64* seconds, int32* nanos) { } else if (*data == '+') { ++data; int64 offset; - if ((data = ParseTimezoneOffset(data, &offset)) == NULL) { + if ((data = ParseTimezoneOffset(data, &offset)) == nullptr) { return false; } *seconds -= offset; } else if (*data == '-') { ++data; int64 offset; - if ((data = ParseTimezoneOffset(data, &offset)) == NULL) { + if ((data = ParseTimezoneOffset(data, &offset)) == nullptr) { return false; } *seconds += offset; diff --git a/third_party/protobuf-lite/wire_format_lite.cc b/third_party/protobuf-lite/wire_format_lite.cc index 1d8cda5a..47ad0f1e 100644 --- a/third_party/protobuf-lite/wire_format_lite.cc +++ b/third_party/protobuf-lite/wire_format_lite.cc @@ -32,24 +32,26 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include +#include #include #include #include + #include #include #include -#include +#include #include #include +#include + namespace google { namespace protobuf { namespace internal { - #if !defined(_MSC_VER) || _MSC_VER >= 1900 // Old version of MSVC doesn't like definitions of inline constants, GCC // requires them. @@ -62,64 +64,63 @@ const int WireFormatLite::kMessageSetMessageTag; // IBM xlC requires prefixing constants with WireFormatLite:: const size_t WireFormatLite::kMessageSetItemTagsSize = - io::CodedOutputStream::StaticVarintSize32< - WireFormatLite::kMessageSetItemStartTag>::value + - io::CodedOutputStream::StaticVarintSize32< - WireFormatLite::kMessageSetItemEndTag>::value + - io::CodedOutputStream::StaticVarintSize32< - WireFormatLite::kMessageSetTypeIdTag>::value + - io::CodedOutputStream::StaticVarintSize32< - WireFormatLite::kMessageSetMessageTag>::value; + io::CodedOutputStream::StaticVarintSize32< + WireFormatLite::kMessageSetItemStartTag>::value + + io::CodedOutputStream::StaticVarintSize32< + WireFormatLite::kMessageSetItemEndTag>::value + + io::CodedOutputStream::StaticVarintSize32< + WireFormatLite::kMessageSetTypeIdTag>::value + + io::CodedOutputStream::StaticVarintSize32< + WireFormatLite::kMessageSetMessageTag>::value; const WireFormatLite::CppType -WireFormatLite::kFieldTypeToCppTypeMap[MAX_FIELD_TYPE + 1] = { - static_cast(0), // 0 is reserved for errors - - CPPTYPE_DOUBLE, // TYPE_DOUBLE - CPPTYPE_FLOAT, // TYPE_FLOAT - CPPTYPE_INT64, // TYPE_INT64 - CPPTYPE_UINT64, // TYPE_UINT64 - CPPTYPE_INT32, // TYPE_INT32 - CPPTYPE_UINT64, // TYPE_FIXED64 - CPPTYPE_UINT32, // TYPE_FIXED32 - CPPTYPE_BOOL, // TYPE_BOOL - CPPTYPE_STRING, // TYPE_STRING - CPPTYPE_MESSAGE, // TYPE_GROUP - CPPTYPE_MESSAGE, // TYPE_MESSAGE - CPPTYPE_STRING, // TYPE_BYTES - CPPTYPE_UINT32, // TYPE_UINT32 - CPPTYPE_ENUM, // TYPE_ENUM - CPPTYPE_INT32, // TYPE_SFIXED32 - CPPTYPE_INT64, // TYPE_SFIXED64 - CPPTYPE_INT32, // TYPE_SINT32 - CPPTYPE_INT64, // TYPE_SINT64 + WireFormatLite::kFieldTypeToCppTypeMap[MAX_FIELD_TYPE + 1] = { + static_cast(0), // 0 is reserved for errors + + CPPTYPE_DOUBLE, // TYPE_DOUBLE + CPPTYPE_FLOAT, // TYPE_FLOAT + CPPTYPE_INT64, // TYPE_INT64 + CPPTYPE_UINT64, // TYPE_UINT64 + CPPTYPE_INT32, // TYPE_INT32 + CPPTYPE_UINT64, // TYPE_FIXED64 + CPPTYPE_UINT32, // TYPE_FIXED32 + CPPTYPE_BOOL, // TYPE_BOOL + CPPTYPE_STRING, // TYPE_STRING + CPPTYPE_MESSAGE, // TYPE_GROUP + CPPTYPE_MESSAGE, // TYPE_MESSAGE + CPPTYPE_STRING, // TYPE_BYTES + CPPTYPE_UINT32, // TYPE_UINT32 + CPPTYPE_ENUM, // TYPE_ENUM + CPPTYPE_INT32, // TYPE_SFIXED32 + CPPTYPE_INT64, // TYPE_SFIXED64 + CPPTYPE_INT32, // TYPE_SINT32 + CPPTYPE_INT64, // TYPE_SINT64 }; const WireFormatLite::WireType -WireFormatLite::kWireTypeForFieldType[MAX_FIELD_TYPE + 1] = { - static_cast(-1), // invalid - WireFormatLite::WIRETYPE_FIXED64, // TYPE_DOUBLE - WireFormatLite::WIRETYPE_FIXED32, // TYPE_FLOAT - WireFormatLite::WIRETYPE_VARINT, // TYPE_INT64 - WireFormatLite::WIRETYPE_VARINT, // TYPE_UINT64 - WireFormatLite::WIRETYPE_VARINT, // TYPE_INT32 - WireFormatLite::WIRETYPE_FIXED64, // TYPE_FIXED64 - WireFormatLite::WIRETYPE_FIXED32, // TYPE_FIXED32 - WireFormatLite::WIRETYPE_VARINT, // TYPE_BOOL - WireFormatLite::WIRETYPE_LENGTH_DELIMITED, // TYPE_STRING - WireFormatLite::WIRETYPE_START_GROUP, // TYPE_GROUP - WireFormatLite::WIRETYPE_LENGTH_DELIMITED, // TYPE_MESSAGE - WireFormatLite::WIRETYPE_LENGTH_DELIMITED, // TYPE_BYTES - WireFormatLite::WIRETYPE_VARINT, // TYPE_UINT32 - WireFormatLite::WIRETYPE_VARINT, // TYPE_ENUM - WireFormatLite::WIRETYPE_FIXED32, // TYPE_SFIXED32 - WireFormatLite::WIRETYPE_FIXED64, // TYPE_SFIXED64 - WireFormatLite::WIRETYPE_VARINT, // TYPE_SINT32 - WireFormatLite::WIRETYPE_VARINT, // TYPE_SINT64 + WireFormatLite::kWireTypeForFieldType[MAX_FIELD_TYPE + 1] = { + static_cast(-1), // invalid + WireFormatLite::WIRETYPE_FIXED64, // TYPE_DOUBLE + WireFormatLite::WIRETYPE_FIXED32, // TYPE_FLOAT + WireFormatLite::WIRETYPE_VARINT, // TYPE_INT64 + WireFormatLite::WIRETYPE_VARINT, // TYPE_UINT64 + WireFormatLite::WIRETYPE_VARINT, // TYPE_INT32 + WireFormatLite::WIRETYPE_FIXED64, // TYPE_FIXED64 + WireFormatLite::WIRETYPE_FIXED32, // TYPE_FIXED32 + WireFormatLite::WIRETYPE_VARINT, // TYPE_BOOL + WireFormatLite::WIRETYPE_LENGTH_DELIMITED, // TYPE_STRING + WireFormatLite::WIRETYPE_START_GROUP, // TYPE_GROUP + WireFormatLite::WIRETYPE_LENGTH_DELIMITED, // TYPE_MESSAGE + WireFormatLite::WIRETYPE_LENGTH_DELIMITED, // TYPE_BYTES + WireFormatLite::WIRETYPE_VARINT, // TYPE_UINT32 + WireFormatLite::WIRETYPE_VARINT, // TYPE_ENUM + WireFormatLite::WIRETYPE_FIXED32, // TYPE_SFIXED32 + WireFormatLite::WIRETYPE_FIXED64, // TYPE_SFIXED64 + WireFormatLite::WIRETYPE_VARINT, // TYPE_SINT32 + WireFormatLite::WIRETYPE_VARINT, // TYPE_SINT64 }; -bool WireFormatLite::SkipField( - io::CodedInputStream* input, uint32 tag) { +bool WireFormatLite::SkipField(io::CodedInputStream* input, uint32 tag) { // Field number 0 is illegal. if (WireFormatLite::GetTagFieldNumber(tag) == 0) return false; switch (WireFormatLite::GetTagWireType(tag)) { @@ -144,9 +145,9 @@ bool WireFormatLite::SkipField( if (!SkipMessage(input)) return false; input->DecrementRecursionDepth(); // Check that the ending tag matched the starting tag. - if (!input->LastTagWas(WireFormatLite::MakeTag( - WireFormatLite::GetTagFieldNumber(tag), - WireFormatLite::WIRETYPE_END_GROUP))) { + if (!input->LastTagWas( + WireFormatLite::MakeTag(WireFormatLite::GetTagFieldNumber(tag), + WireFormatLite::WIRETYPE_END_GROUP))) { return false; } return true; @@ -165,8 +166,8 @@ bool WireFormatLite::SkipField( } } -bool WireFormatLite::SkipField( - io::CodedInputStream* input, uint32 tag, io::CodedOutputStream* output) { +bool WireFormatLite::SkipField(io::CodedInputStream* input, uint32 tag, + io::CodedOutputStream* output) { // Field number 0 is illegal. if (WireFormatLite::GetTagFieldNumber(tag) == 0) return false; switch (WireFormatLite::GetTagWireType(tag)) { @@ -190,7 +191,7 @@ bool WireFormatLite::SkipField( output->WriteVarint32(tag); output->WriteVarint32(length); // TODO(mkilavuz): Provide API to prevent extra string copying. - string temp; + std::string temp; if (!input->ReadString(&temp, length)) return false; output->WriteString(temp); return true; @@ -201,9 +202,9 @@ bool WireFormatLite::SkipField( if (!SkipMessage(input, output)) return false; input->DecrementRecursionDepth(); // Check that the ending tag matched the starting tag. - if (!input->LastTagWas(WireFormatLite::MakeTag( - WireFormatLite::GetTagFieldNumber(tag), - WireFormatLite::WIRETYPE_END_GROUP))) { + if (!input->LastTagWas( + WireFormatLite::MakeTag(WireFormatLite::GetTagFieldNumber(tag), + WireFormatLite::WIRETYPE_END_GROUP))) { return false; } return true; @@ -244,7 +245,7 @@ bool WireFormatLite::SkipMessage(io::CodedInputStream* input) { } bool WireFormatLite::SkipMessage(io::CodedInputStream* input, - io::CodedOutputStream* output) { + io::CodedOutputStream* output) { while (true) { uint32 tag = input->ReadTag(); if (tag == 0) { @@ -264,8 +265,7 @@ bool WireFormatLite::SkipMessage(io::CodedInputStream* input, } } -bool FieldSkipper::SkipField( - io::CodedInputStream* input, uint32 tag) { +bool FieldSkipper::SkipField(io::CodedInputStream* input, uint32 tag) { return WireFormatLite::SkipField(input, tag); } @@ -273,13 +273,12 @@ bool FieldSkipper::SkipMessage(io::CodedInputStream* input) { return WireFormatLite::SkipMessage(input); } -void FieldSkipper::SkipUnknownEnum( - int /* field_number */, int /* value */) { +void FieldSkipper::SkipUnknownEnum(int /* field_number */, int /* value */) { // Nothing. } -bool CodedOutputStreamFieldSkipper::SkipField( - io::CodedInputStream* input, uint32 tag) { +bool CodedOutputStreamFieldSkipper::SkipField(io::CodedInputStream* input, + uint32 tag) { return WireFormatLite::SkipField(input, tag, unknown_fields_); } @@ -287,45 +286,21 @@ bool CodedOutputStreamFieldSkipper::SkipMessage(io::CodedInputStream* input) { return WireFormatLite::SkipMessage(input, unknown_fields_); } -void CodedOutputStreamFieldSkipper::SkipUnknownEnum( - int field_number, int value) { +void CodedOutputStreamFieldSkipper::SkipUnknownEnum(int field_number, + int value) { unknown_fields_->WriteVarint32(field_number); unknown_fields_->WriteVarint64(value); } -bool WireFormatLite::ReadPackedEnumNoInline(io::CodedInputStream* input, - bool (*is_valid)(int), - RepeatedField* values) { - uint32 length; - if (!input->ReadVarint32(&length)) return false; - io::CodedInputStream::Limit limit = input->PushLimit(length); - while (input->BytesUntilLimit() > 0) { - int value; - if (!google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, WireFormatLite::TYPE_ENUM>(input, &value)) { - return false; - } - if (is_valid == NULL || is_valid(value)) { - values->Add(value); - } - } - input->PopLimit(limit); - return true; -} - bool WireFormatLite::ReadPackedEnumPreserveUnknowns( - io::CodedInputStream* input, - int field_number, - bool (*is_valid)(int), - io::CodedOutputStream* unknown_fields_stream, - RepeatedField* values) { + io::CodedInputStream* input, int field_number, bool (*is_valid)(int), + io::CodedOutputStream* unknown_fields_stream, RepeatedField* values) { uint32 length; if (!input->ReadVarint32(&length)) return false; io::CodedInputStream::Limit limit = input->PushLimit(length); while (input->BytesUntilLimit() > 0) { int value; - if (!google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, WireFormatLite::TYPE_ENUM>(input, &value)) { + if (!ReadPrimitive(input, &value)) { return false; } if (is_valid == NULL || is_valid(value)) { @@ -386,7 +361,7 @@ static void WriteArray(const CType* a, int n, io::CodedOutputStream* output) { int to_do = std::min(kAtATime, n - i); uint8* ptr = buf; for (int j = 0; j < to_do; j++) { - EncodeFixedSizeValue(a[i+j], ptr); + EncodeFixedSizeValue(a[i + j], ptr); ptr += sizeof(a[0]); } output->WriteRaw(buf, to_do * sizeof(a[0])); @@ -400,12 +375,12 @@ void WireFormatLite::WriteFloatArray(const float* a, int n, } void WireFormatLite::WriteDoubleArray(const double* a, int n, - io::CodedOutputStream* output) { + io::CodedOutputStream* output) { WriteArray(a, n, output); } void WireFormatLite::WriteFixed32Array(const uint32* a, int n, - io::CodedOutputStream* output) { + io::CodedOutputStream* output) { WriteArray(a, n, output); } @@ -415,12 +390,12 @@ void WireFormatLite::WriteFixed64Array(const uint64* a, int n, } void WireFormatLite::WriteSFixed32Array(const int32* a, int n, - io::CodedOutputStream* output) { + io::CodedOutputStream* output) { WriteArray(a, n, output); } void WireFormatLite::WriteSFixed64Array(const int64* a, int n, - io::CodedOutputStream* output) { + io::CodedOutputStream* output) { WriteArray(a, n, output); } @@ -500,7 +475,7 @@ void WireFormatLite::WriteEnum(int field_number, int value, WriteEnumNoTag(value, output); } -void WireFormatLite::WriteString(int field_number, const string& value, +void WireFormatLite::WriteString(int field_number, const std::string& value, io::CodedOutputStream* output) { // String is for UTF-8 text only WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); @@ -508,25 +483,25 @@ void WireFormatLite::WriteString(int field_number, const string& value, output->WriteVarint32(value.size()); output->WriteString(value); } -void WireFormatLite::WriteStringMaybeAliased( - int field_number, const string& value, - io::CodedOutputStream* output) { +void WireFormatLite::WriteStringMaybeAliased(int field_number, + const std::string& value, + io::CodedOutputStream* output) { // String is for UTF-8 text only WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); GOOGLE_CHECK_LE(value.size(), kint32max); output->WriteVarint32(value.size()); output->WriteRawMaybeAliased(value.data(), value.size()); } -void WireFormatLite::WriteBytes(int field_number, const string& value, +void WireFormatLite::WriteBytes(int field_number, const std::string& value, io::CodedOutputStream* output) { WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); GOOGLE_CHECK_LE(value.size(), kint32max); output->WriteVarint32(value.size()); output->WriteString(value); } -void WireFormatLite::WriteBytesMaybeAliased( - int field_number, const string& value, - io::CodedOutputStream* output) { +void WireFormatLite::WriteBytesMaybeAliased(int field_number, + const std::string& value, + io::CodedOutputStream* output) { WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); GOOGLE_CHECK_LE(value.size(), kint32max); output->WriteVarint32(value.size()); @@ -534,16 +509,14 @@ void WireFormatLite::WriteBytesMaybeAliased( } -void WireFormatLite::WriteGroup(int field_number, - const MessageLite& value, +void WireFormatLite::WriteGroup(int field_number, const MessageLite& value, io::CodedOutputStream* output) { WriteTag(field_number, WIRETYPE_START_GROUP, output); value.SerializeWithCachedSizes(output); WriteTag(field_number, WIRETYPE_END_GROUP, output); } -void WireFormatLite::WriteMessage(int field_number, - const MessageLite& value, +void WireFormatLite::WriteMessage(int field_number, const MessageLite& value, io::CodedOutputStream* output) { WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); const int size = value.GetCachedSize(); @@ -551,19 +524,17 @@ void WireFormatLite::WriteMessage(int field_number, value.SerializeWithCachedSizes(output); } +void WireFormatLite::WriteSubMessageMaybeToArray( + int size, const MessageLite& value, io::CodedOutputStream* output) { + output->SetCur(value._InternalSerialize(output->Cur(), output->EpsCopy())); +} + void WireFormatLite::WriteGroupMaybeToArray(int field_number, const MessageLite& value, io::CodedOutputStream* output) { WriteTag(field_number, WIRETYPE_START_GROUP, output); const int size = value.GetCachedSize(); - uint8* target = output->GetDirectBufferForNBytesAndAdvance(size); - if (target != NULL) { - uint8* end = value.InternalSerializeWithCachedSizesToArray( - output->IsSerializationDeterministic(), target); - GOOGLE_DCHECK_EQ(end - target, size); - } else { - value.SerializeWithCachedSizes(output); - } + WriteSubMessageMaybeToArray(size, value, output); WriteTag(field_number, WIRETYPE_END_GROUP, output); } @@ -573,39 +544,43 @@ void WireFormatLite::WriteMessageMaybeToArray(int field_number, WriteTag(field_number, WIRETYPE_LENGTH_DELIMITED, output); const int size = value.GetCachedSize(); output->WriteVarint32(size); - uint8* target = output->GetDirectBufferForNBytesAndAdvance(size); - if (target != NULL) { - uint8* end = value.InternalSerializeWithCachedSizesToArray( - output->IsSerializationDeterministic(), target); - GOOGLE_DCHECK_EQ(end - target, size); - } else { - value.SerializeWithCachedSizes(output); - } + WriteSubMessageMaybeToArray(size, value, output); } -GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static bool ReadBytesToString( - io::CodedInputStream* input, string* value); +PROTOBUF_ALWAYS_INLINE static bool ReadBytesToString( + io::CodedInputStream* input, std::string* value); inline static bool ReadBytesToString(io::CodedInputStream* input, - string* value) { + std::string* value) { uint32 length; - return input->ReadVarint32(&length) && - input->InternalReadStringInline(value, length); + return input->ReadVarint32(&length) && input->ReadString(value, length); } -bool WireFormatLite::ReadBytes(io::CodedInputStream* input, string* value) { +bool WireFormatLite::ReadBytes(io::CodedInputStream* input, + std::string* value) { return ReadBytesToString(input, value); } -bool WireFormatLite::ReadBytes(io::CodedInputStream* input, string** p) { - if (*p == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - *p = new ::std::string(); +bool WireFormatLite::ReadBytes(io::CodedInputStream* input, std::string** p) { + if (*p == &GetEmptyStringAlreadyInited()) { + *p = new std::string(); } return ReadBytesToString(input, *p); } -bool WireFormatLite::VerifyUtf8String(const char* data, - int size, - Operation op, +void PrintUTF8ErrorLog(const char* field_name, const char* operation_str, + bool emit_stacktrace) { + std::string stacktrace; + std::string quoted_field_name = ""; + if (field_name != nullptr) { + quoted_field_name = StringPrintf(" '%s'", field_name); + } + GOOGLE_LOG(ERROR) << "String field" << quoted_field_name << " contains invalid " + << "UTF-8 data when " << operation_str << " a protocol " + << "buffer. Use the 'bytes' type if you intend to send raw " + << "bytes. " << stacktrace; +} + +bool WireFormatLite::VerifyUtf8String(const char* data, int size, Operation op, const char* field_name) { if (!IsStructurallyValidUTF8(data, size)) { const char* operation_str = NULL; @@ -616,17 +591,9 @@ bool WireFormatLite::VerifyUtf8String(const char* data, case SERIALIZE: operation_str = "serializing"; break; - // no default case: have the compiler warn if a case is not covered. + // no default case: have the compiler warn if a case is not covered. } - string quoted_field_name = ""; - if (field_name != NULL) { - quoted_field_name = StringPrintf(" '%s'", field_name); - } - // no space below to avoid double space when the field name is missing. - GOOGLE_LOG(ERROR) << "String field" << quoted_field_name << " contains invalid " - << "UTF-8 data when " << operation_str << " a protocol " - << "buffer. Use the 'bytes' type if you intend to send raw " - << "bytes. "; + PrintUTF8ErrorLog(field_name, operation_str, false); return false; } return true; @@ -634,18 +601,18 @@ bool WireFormatLite::VerifyUtf8String(const char* data, // this code is deliberately written such that clang makes it into really // efficient SSE code. -template +template static size_t VarintSize(const T* data, const int n) { #if __cplusplus >= 201103L static_assert(sizeof(T) == 4, "This routine only works for 32 bit integers"); // is_unsigned => !ZigZag - static_assert((std::is_unsigned::value ^ ZigZag) || - std::is_signed::value, - "Cannot ZigZag encode unsigned types"); + static_assert( + (std::is_unsigned::value ^ ZigZag) || std::is_signed::value, + "Cannot ZigZag encode unsigned types"); // is_unsigned => !SignExtended - static_assert((std::is_unsigned::value ^ SignExtended) || - std::is_signed::value, - "Cannot SignExtended unsigned types"); + static_assert( + (std::is_unsigned::value ^ SignExtended) || std::is_signed::value, + "Cannot SignExtended unsigned types"); static_assert(!(SignExtended && ZigZag), "Cannot SignExtended and ZigZag on the same type"); #endif @@ -671,7 +638,7 @@ static size_t VarintSize(const T* data, const int n) { return sum; } -template +template static size_t VarintSize64(const T* data, const int n) { #if __cplusplus >= 201103L static_assert(sizeof(T) == 8, "This routine only works for 64 bit integers"); @@ -767,7 +734,7 @@ size_t WireFormatLite::EnumSize(const RepeatedField& value) { // enable this. #define USE_SSE_FOR_64_BIT_INTEGER_ARRAYS 0 #if USE_SSE_FOR_64_BIT_INTEGER_ARRAYS -size_t WireFormatLite::Int64Size (const RepeatedField< int64>& value) { +size_t WireFormatLite::Int64Size(const RepeatedField& value) { return VarintSize64(value.data(), value.size()); } @@ -775,13 +742,13 @@ size_t WireFormatLite::UInt64Size(const RepeatedField& value) { return VarintSize64(value.data(), value.size()); } -size_t WireFormatLite::SInt64Size(const RepeatedField< int64>& value) { +size_t WireFormatLite::SInt64Size(const RepeatedField& value) { return VarintSize64(value.data(), value.size()); } #else -size_t WireFormatLite::Int64Size (const RepeatedField< int64>& value) { +size_t WireFormatLite::Int64Size(const RepeatedField& value) { size_t out = 0; const int n = value.size(); for (int i = 0; i < n; i++) { @@ -799,7 +766,7 @@ size_t WireFormatLite::UInt64Size(const RepeatedField& value) { return out; } -size_t WireFormatLite::SInt64Size(const RepeatedField< int64>& value) { +size_t WireFormatLite::SInt64Size(const RepeatedField& value) { size_t out = 0; const int n = value.size(); for (int i = 0; i < n; i++) { diff --git a/third_party/protobuf-lite/zero_copy_stream_impl.cc b/third_party/protobuf-lite/zero_copy_stream_impl.cc new file mode 100644 index 00000000..4b1bf802 --- /dev/null +++ b/third_party/protobuf-lite/zero_copy_stream_impl.cc @@ -0,0 +1,375 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#ifndef _MSC_VER +#include +#include +#include +#include +#endif +#include + +#include +#include + +#include +#include +#include +#include +#include + + +namespace google { +namespace protobuf { +namespace io { + +#ifdef _WIN32 +// Win32 lseek is broken: If invoked on a non-seekable file descriptor, its +// return value is undefined. We re-define it to always produce an error. +#define lseek(fd, offset, origin) ((off_t)-1) +// DO NOT include , instead create functions in io_win32.{h,cc} and import +// them like we do below. +using google::protobuf::io::win32::access; +using google::protobuf::io::win32::close; +using google::protobuf::io::win32::open; +using google::protobuf::io::win32::read; +using google::protobuf::io::win32::write; +#endif + +namespace { + +// EINTR sucks. +int close_no_eintr(int fd) { + int result; + do { + result = close(fd); + } while (result < 0 && errno == EINTR); + return result; +} + +} // namespace + +// =================================================================== + +FileInputStream::FileInputStream(int file_descriptor, int block_size) + : copying_input_(file_descriptor), impl_(©ing_input_, block_size) {} + +bool FileInputStream::Close() { return copying_input_.Close(); } + +bool FileInputStream::Next(const void** data, int* size) { + return impl_.Next(data, size); +} + +void FileInputStream::BackUp(int count) { impl_.BackUp(count); } + +bool FileInputStream::Skip(int count) { return impl_.Skip(count); } + +int64_t FileInputStream::ByteCount() const { return impl_.ByteCount(); } + +FileInputStream::CopyingFileInputStream::CopyingFileInputStream( + int file_descriptor) + : file_(file_descriptor), + close_on_delete_(false), + is_closed_(false), + errno_(0), + previous_seek_failed_(false) {} + +FileInputStream::CopyingFileInputStream::~CopyingFileInputStream() { + if (close_on_delete_) { + if (!Close()) { + GOOGLE_LOG(ERROR) << "close() failed: " << strerror(errno_); + } + } +} + +bool FileInputStream::CopyingFileInputStream::Close() { + GOOGLE_CHECK(!is_closed_); + + is_closed_ = true; + if (close_no_eintr(file_) != 0) { + // The docs on close() do not specify whether a file descriptor is still + // open after close() fails with EIO. However, the glibc source code + // seems to indicate that it is not. + errno_ = errno; + return false; + } + + return true; +} + +int FileInputStream::CopyingFileInputStream::Read(void* buffer, int size) { + GOOGLE_CHECK(!is_closed_); + + int result; + do { + result = read(file_, buffer, size); + } while (result < 0 && errno == EINTR); + + if (result < 0) { + // Read error (not EOF). + errno_ = errno; + } + + return result; +} + +int FileInputStream::CopyingFileInputStream::Skip(int count) { + GOOGLE_CHECK(!is_closed_); + + if (!previous_seek_failed_ && lseek(file_, count, SEEK_CUR) != (off_t)-1) { + // Seek succeeded. + return count; + } else { + // Failed to seek. + + // Note to self: Don't seek again. This file descriptor doesn't + // support it. + previous_seek_failed_ = true; + + // Use the default implementation. + return CopyingInputStream::Skip(count); + } +} + +// =================================================================== + +FileOutputStream::FileOutputStream(int file_descriptor, int block_size) + : copying_output_(file_descriptor), impl_(©ing_output_, block_size) {} + +FileOutputStream::~FileOutputStream() { impl_.Flush(); } + +bool FileOutputStream::Close() { + bool flush_succeeded = impl_.Flush(); + return copying_output_.Close() && flush_succeeded; +} + +bool FileOutputStream::Flush() { return impl_.Flush(); } + +bool FileOutputStream::Next(void** data, int* size) { + return impl_.Next(data, size); +} + +void FileOutputStream::BackUp(int count) { impl_.BackUp(count); } + +int64_t FileOutputStream::ByteCount() const { return impl_.ByteCount(); } + +FileOutputStream::CopyingFileOutputStream::CopyingFileOutputStream( + int file_descriptor) + : file_(file_descriptor), + close_on_delete_(false), + is_closed_(false), + errno_(0) {} + +FileOutputStream::CopyingFileOutputStream::~CopyingFileOutputStream() { + if (close_on_delete_) { + if (!Close()) { + GOOGLE_LOG(ERROR) << "close() failed: " << strerror(errno_); + } + } +} + +bool FileOutputStream::CopyingFileOutputStream::Close() { + GOOGLE_CHECK(!is_closed_); + + is_closed_ = true; + if (close_no_eintr(file_) != 0) { + // The docs on close() do not specify whether a file descriptor is still + // open after close() fails with EIO. However, the glibc source code + // seems to indicate that it is not. + errno_ = errno; + return false; + } + + return true; +} + +bool FileOutputStream::CopyingFileOutputStream::Write(const void* buffer, + int size) { + GOOGLE_CHECK(!is_closed_); + int total_written = 0; + + const uint8* buffer_base = reinterpret_cast(buffer); + + while (total_written < size) { + int bytes; + do { + bytes = write(file_, buffer_base + total_written, size - total_written); + } while (bytes < 0 && errno == EINTR); + + if (bytes <= 0) { + // Write error. + + // FIXME(kenton): According to the man page, if write() returns zero, + // there was no error; write() simply did not write anything. It's + // unclear under what circumstances this might happen, but presumably + // errno won't be set in this case. I am confused as to how such an + // event should be handled. For now I'm treating it as an error, since + // retrying seems like it could lead to an infinite loop. I suspect + // this never actually happens anyway. + + if (bytes < 0) { + errno_ = errno; + } + return false; + } + total_written += bytes; + } + + return true; +} + +// =================================================================== + +IstreamInputStream::IstreamInputStream(std::istream* input, int block_size) + : copying_input_(input), impl_(©ing_input_, block_size) {} + +bool IstreamInputStream::Next(const void** data, int* size) { + return impl_.Next(data, size); +} + +void IstreamInputStream::BackUp(int count) { impl_.BackUp(count); } + +bool IstreamInputStream::Skip(int count) { return impl_.Skip(count); } + +int64_t IstreamInputStream::ByteCount() const { return impl_.ByteCount(); } + +IstreamInputStream::CopyingIstreamInputStream::CopyingIstreamInputStream( + std::istream* input) + : input_(input) {} + +IstreamInputStream::CopyingIstreamInputStream::~CopyingIstreamInputStream() {} + +int IstreamInputStream::CopyingIstreamInputStream::Read(void* buffer, + int size) { + input_->read(reinterpret_cast(buffer), size); + int result = input_->gcount(); + if (result == 0 && input_->fail() && !input_->eof()) { + return -1; + } + return result; +} + +// =================================================================== + +OstreamOutputStream::OstreamOutputStream(std::ostream* output, int block_size) + : copying_output_(output), impl_(©ing_output_, block_size) {} + +OstreamOutputStream::~OstreamOutputStream() { impl_.Flush(); } + +bool OstreamOutputStream::Next(void** data, int* size) { + return impl_.Next(data, size); +} + +void OstreamOutputStream::BackUp(int count) { impl_.BackUp(count); } + +int64_t OstreamOutputStream::ByteCount() const { return impl_.ByteCount(); } + +OstreamOutputStream::CopyingOstreamOutputStream::CopyingOstreamOutputStream( + std::ostream* output) + : output_(output) {} + +OstreamOutputStream::CopyingOstreamOutputStream::~CopyingOstreamOutputStream() { +} + +bool OstreamOutputStream::CopyingOstreamOutputStream::Write(const void* buffer, + int size) { + output_->write(reinterpret_cast(buffer), size); + return output_->good(); +} + +// =================================================================== + +ConcatenatingInputStream::ConcatenatingInputStream( + ZeroCopyInputStream* const streams[], int count) + : streams_(streams), stream_count_(count), bytes_retired_(0) { +} + +bool ConcatenatingInputStream::Next(const void** data, int* size) { + while (stream_count_ > 0) { + if (streams_[0]->Next(data, size)) return true; + + // That stream is done. Advance to the next one. + bytes_retired_ += streams_[0]->ByteCount(); + ++streams_; + --stream_count_; + } + + // No more streams. + return false; +} + +void ConcatenatingInputStream::BackUp(int count) { + if (stream_count_ > 0) { + streams_[0]->BackUp(count); + } else { + GOOGLE_LOG(DFATAL) << "Can't BackUp() after failed Next()."; + } +} + +bool ConcatenatingInputStream::Skip(int count) { + while (stream_count_ > 0) { + // Assume that ByteCount() can be used to find out how much we actually + // skipped when Skip() fails. + int64 target_byte_count = streams_[0]->ByteCount() + count; + if (streams_[0]->Skip(count)) return true; + + // Hit the end of the stream. Figure out how many more bytes we still have + // to skip. + int64 final_byte_count = streams_[0]->ByteCount(); + GOOGLE_DCHECK_LT(final_byte_count, target_byte_count); + count = target_byte_count - final_byte_count; + + // That stream is done. Advance to the next one. + bytes_retired_ += final_byte_count; + ++streams_; + --stream_count_; + } + + return false; +} + +int64_t ConcatenatingInputStream::ByteCount() const { + if (stream_count_ == 0) { + return bytes_retired_; + } else { + return bytes_retired_ + streams_[0]->ByteCount(); + } +} + + +// =================================================================== + +} // namespace io +} // namespace protobuf +} // namespace google diff --git a/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc b/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc index 66ad49bc..51cda2a8 100644 --- a/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc +++ b/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc @@ -37,9 +37,9 @@ #include #include -#include #include #include +#include #include namespace google { @@ -55,14 +55,12 @@ static const int kDefaultBlockSize = 8192; // =================================================================== -ArrayInputStream::ArrayInputStream(const void* data, int size, - int block_size) - : data_(reinterpret_cast(data)), - size_(size), - block_size_(block_size > 0 ? block_size : size), - position_(0), - last_returned_size_(0) { -} +ArrayInputStream::ArrayInputStream(const void* data, int size, int block_size) + : data_(reinterpret_cast(data)), + size_(size), + block_size_(block_size > 0 ? block_size : size), + position_(0), + last_returned_size_(0) {} bool ArrayInputStream::Next(const void** data, int* size) { if (position_ < size_) { @@ -73,7 +71,7 @@ bool ArrayInputStream::Next(const void** data, int* size) { return true; } else { // We're at the end of the array. - last_returned_size_ = 0; // Don't let caller back up. + last_returned_size_ = 0; // Don't let caller back up. return false; } } @@ -89,7 +87,7 @@ void ArrayInputStream::BackUp(int count) { bool ArrayInputStream::Skip(int count) { GOOGLE_CHECK_GE(count, 0); - last_returned_size_ = 0; // Don't let caller back up. + last_returned_size_ = 0; // Don't let caller back up. if (count > size_ - position_) { position_ = size_; return false; @@ -99,20 +97,17 @@ bool ArrayInputStream::Skip(int count) { } } -int64 ArrayInputStream::ByteCount() const { - return position_; -} +int64_t ArrayInputStream::ByteCount() const { return position_; } // =================================================================== ArrayOutputStream::ArrayOutputStream(void* data, int size, int block_size) - : data_(reinterpret_cast(data)), - size_(size), - block_size_(block_size > 0 ? block_size : size), - position_(0), - last_returned_size_(0) { -} + : data_(reinterpret_cast(data)), + size_(size), + block_size_(block_size > 0 ? block_size : size), + position_(0), + last_returned_size_(0) {} bool ArrayOutputStream::Next(void** data, int* size) { if (position_ < size_) { @@ -123,7 +118,7 @@ bool ArrayOutputStream::Next(void** data, int* size) { return true; } else { // We're at the end of the array. - last_returned_size_ = 0; // Don't let caller back up. + last_returned_size_ = 0; // Don't let caller back up. return false; } } @@ -137,15 +132,11 @@ void ArrayOutputStream::BackUp(int count) { last_returned_size_ = 0; // Don't let caller back up further. } -int64 ArrayOutputStream::ByteCount() const { - return position_; -} +int64_t ArrayOutputStream::ByteCount() const { return position_; } // =================================================================== -StringOutputStream::StringOutputStream(string* target) - : target_(target) { -} +StringOutputStream::StringOutputStream(std::string* target) : target_(target) {} bool StringOutputStream::Next(void** data, int* size) { GOOGLE_CHECK(target_ != NULL); @@ -185,23 +176,19 @@ void StringOutputStream::BackUp(int count) { target_->resize(target_->size() - count); } -int64 StringOutputStream::ByteCount() const { +int64_t StringOutputStream::ByteCount() const { GOOGLE_CHECK(target_ != NULL); return target_->size(); } -void StringOutputStream::SetString(string* target) { - target_ = target; -} - // =================================================================== int CopyingInputStream::Skip(int count) { char junk[4096]; int skipped = 0; while (skipped < count) { - int bytes = - Read(junk, std::min(count - skipped, implicit_cast(sizeof(junk)))); + int bytes = Read(junk, std::min(count - skipped, + implicit_cast(sizeof(junk)))); if (bytes <= 0) { // EOF or read error. return skipped; @@ -213,14 +200,13 @@ int CopyingInputStream::Skip(int count) { CopyingInputStreamAdaptor::CopyingInputStreamAdaptor( CopyingInputStream* copying_stream, int block_size) - : copying_stream_(copying_stream), - owns_copying_stream_(false), - failed_(false), - position_(0), - buffer_size_(block_size > 0 ? block_size : kDefaultBlockSize), - buffer_used_(0), - backup_bytes_(0) { -} + : copying_stream_(copying_stream), + owns_copying_stream_(false), + failed_(false), + position_(0), + buffer_size_(block_size > 0 ? block_size : kDefaultBlockSize), + buffer_used_(0), + backup_bytes_(0) {} CopyingInputStreamAdaptor::~CopyingInputStreamAdaptor() { if (owns_copying_stream_) { @@ -264,12 +250,11 @@ bool CopyingInputStreamAdaptor::Next(const void** data, int* size) { void CopyingInputStreamAdaptor::BackUp(int count) { GOOGLE_CHECK(backup_bytes_ == 0 && buffer_.get() != NULL) - << " BackUp() can only be called after Next()."; + << " BackUp() can only be called after Next()."; GOOGLE_CHECK_LE(count, buffer_used_) - << " Can't back up over more bytes than were returned by the last call" - " to Next()."; - GOOGLE_CHECK_GE(count, 0) - << " Parameter to BackUp() can't be negative."; + << " Can't back up over more bytes than were returned by the last call" + " to Next()."; + GOOGLE_CHECK_GE(count, 0) << " Parameter to BackUp() can't be negative."; backup_bytes_ = count; } @@ -297,7 +282,7 @@ bool CopyingInputStreamAdaptor::Skip(int count) { return skipped == count; } -int64 CopyingInputStreamAdaptor::ByteCount() const { +int64_t CopyingInputStreamAdaptor::ByteCount() const { return position_ - backup_bytes_; } @@ -317,13 +302,12 @@ void CopyingInputStreamAdaptor::FreeBuffer() { CopyingOutputStreamAdaptor::CopyingOutputStreamAdaptor( CopyingOutputStream* copying_stream, int block_size) - : copying_stream_(copying_stream), - owns_copying_stream_(false), - failed_(false), - position_(0), - buffer_size_(block_size > 0 ? block_size : kDefaultBlockSize), - buffer_used_(0) { -} + : copying_stream_(copying_stream), + owns_copying_stream_(false), + failed_(false), + position_(0), + buffer_size_(block_size > 0 ? block_size : kDefaultBlockSize), + buffer_used_(0) {} CopyingOutputStreamAdaptor::~CopyingOutputStreamAdaptor() { WriteBuffer(); @@ -332,9 +316,7 @@ CopyingOutputStreamAdaptor::~CopyingOutputStreamAdaptor() { } } -bool CopyingOutputStreamAdaptor::Flush() { - return WriteBuffer(); -} +bool CopyingOutputStreamAdaptor::Flush() { return WriteBuffer(); } bool CopyingOutputStreamAdaptor::Next(void** data, int* size) { if (buffer_used_ == buffer_size_) { @@ -352,15 +334,15 @@ bool CopyingOutputStreamAdaptor::Next(void** data, int* size) { void CopyingOutputStreamAdaptor::BackUp(int count) { GOOGLE_CHECK_GE(count, 0); GOOGLE_CHECK_EQ(buffer_used_, buffer_size_) - << " BackUp() can only be called after Next()."; + << " BackUp() can only be called after Next()."; GOOGLE_CHECK_LE(count, buffer_used_) - << " Can't back up over more bytes than were returned by the last call" - " to Next()."; + << " Can't back up over more bytes than were returned by the last call" + " to Next()."; buffer_used_ -= count; } -int64 CopyingOutputStreamAdaptor::ByteCount() const { +int64_t CopyingOutputStreamAdaptor::ByteCount() const { return position_ + buffer_used_; } @@ -394,6 +376,63 @@ void CopyingOutputStreamAdaptor::FreeBuffer() { buffer_.reset(); } +// =================================================================== + +LimitingInputStream::LimitingInputStream(ZeroCopyInputStream* input, + int64 limit) + : input_(input), limit_(limit) { + prior_bytes_read_ = input_->ByteCount(); +} + +LimitingInputStream::~LimitingInputStream() { + // If we overshot the limit, back up. + if (limit_ < 0) input_->BackUp(-limit_); +} + +bool LimitingInputStream::Next(const void** data, int* size) { + if (limit_ <= 0) return false; + if (!input_->Next(data, size)) return false; + + limit_ -= *size; + if (limit_ < 0) { + // We overshot the limit. Reduce *size to hide the rest of the buffer. + *size += limit_; + } + return true; +} + +void LimitingInputStream::BackUp(int count) { + if (limit_ < 0) { + input_->BackUp(count - limit_); + limit_ = count; + } else { + input_->BackUp(count); + limit_ += count; + } +} + +bool LimitingInputStream::Skip(int count) { + if (count > limit_) { + if (limit_ < 0) return false; + input_->Skip(limit_); + limit_ = 0; + return false; + } else { + if (!input_->Skip(count)) return false; + limit_ -= count; + return true; + } +} + +int64_t LimitingInputStream::ByteCount() const { + if (limit_ < 0) { + return input_->ByteCount() + limit_ - prior_bytes_read_; + } else { + return input_->ByteCount() - prior_bytes_read_; + } +} + + // =================================================================== } // namespace io