From 707414b25914e29c6ad7f184b0cbc9d61b5acbbb Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 14 Jul 2025 09:43:31 -0700 Subject: [PATCH] Update `google.protobuf.Type` handling in `protobuf/json` to support editions PiperOrigin-RevId: 782939203 --- src/google/protobuf/descriptor.cc | 44 ++++-- src/google/protobuf/descriptor.h | 13 ++ src/google/protobuf/feature_resolver.h | 7 - src/google/protobuf/internal_feature_helper.h | 6 + .../json/internal/descriptor_traits.h | 25 +++- .../protobuf/json/internal/untyped_message.cc | 94 ++++++++++++ .../protobuf/json/internal/untyped_message.h | 18 +++ .../protobuf/util/type_resolver_util.cc | 138 ++++++++++++++---- 8 files changed, 297 insertions(+), 48 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 91832461edc71..786c8b135395e 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -5190,17 +5190,7 @@ bool DescriptorPool::ShouldEnforceExtensionDeclaration( const FeatureSetDefaults& DescriptorPool::GetFeatureSetDefaults() const { if (feature_set_defaults_spec_ != nullptr) return *feature_set_defaults_spec_; - static const FeatureSetDefaults* cpp_default_spec = - internal::OnShutdownDelete([] { - auto* defaults = new FeatureSetDefaults(); - internal::ParseNoReflection( - absl::string_view{ - PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS, - sizeof(PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS) - 1}, - *defaults); - return defaults; - }()); - return *cpp_default_spec; + return internal::cpp::GetFeatureSetDefaults(); } bool DescriptorPool::ResolvesFeaturesForImpl(int extension_number) const { @@ -10671,6 +10661,18 @@ bool ParseNoReflection(absl::string_view from, google::protobuf::MessageLite& to return to.IsInitializedWithErrors(); } +const FeatureSet& GetUnresolvedFeatureSet(const FileDescriptor& descriptor) { + return InternalFeatureHelper::GetUnresolvedFeatures(descriptor); +} + +const FeatureSet& GetUnresolvedFeatureSet(const Descriptor& descriptor) { + return InternalFeatureHelper::GetUnresolvedFeatures(descriptor); +} + +const FeatureSet& GetUnresolvedFeatureSet(const EnumDescriptor& descriptor) { + return InternalFeatureHelper::GetUnresolvedFeatures(descriptor); +} + namespace cpp { bool HasPreservingUnknownEnumSemantics(const FieldDescriptor* field) { if (field->legacy_enum_field_treated_as_closed()) { @@ -10779,9 +10781,29 @@ bool IsStringFieldWithPrivatizedAccessors(const FieldDescriptor& field) { Edition FileDescriptor::edition() const { return edition_; } namespace internal { + absl::string_view ShortEditionName(Edition edition) { return absl::StripPrefix(Edition_Name(edition), "EDITION_"); } + +namespace cpp { + +const FeatureSetDefaults& GetFeatureSetDefaults() { + static const FeatureSetDefaults* cpp_default_spec = + internal::OnShutdownDelete([] { + auto* defaults = new FeatureSetDefaults(); + internal::ParseNoReflection( + absl::string_view{ + PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS, + sizeof(PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS) - 1}, + *defaults); + return defaults; + }()); + return *cpp_default_spec; +} + +} // namespace cpp + } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index bf07bb2816b9f..bd91a4128058e 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -3144,11 +3144,24 @@ struct FieldRangeImpl { // parsing because that uses reflection to verify consistency. bool ParseNoReflection(absl::string_view from, google::protobuf::MessageLite& to); +PROTOBUF_EXPORT const FeatureSet& GetUnresolvedFeatureSet( + const FileDescriptor& descriptor); +PROTOBUF_EXPORT const FeatureSet& GetUnresolvedFeatureSet( + const Descriptor& descriptor); +PROTOBUF_EXPORT const FeatureSet& GetUnresolvedFeatureSet( + const EnumDescriptor& descriptor); + +// Gets the default feature set for a given edition. +absl::StatusOr PROTOBUF_EXPORT GetEditionFeatureSetDefaults( + Edition edition, const FeatureSetDefaults& defaults); + // The context for these functions under `cpp` is "for the C++ implementation". // In particular, questions like "does this field have a has bit?" have a // different answer depending on the language. namespace cpp { +PROTOBUF_EXPORT const FeatureSetDefaults& GetFeatureSetDefaults(); + // The maximum allowed nesting for message declarations. // Going over this limit will make the proto definition invalid. constexpr int MaxMessageDeclarationNestingDepth() { return 32; } diff --git a/src/google/protobuf/feature_resolver.h b/src/google/protobuf/feature_resolver.h index d66c156e2f66d..ec744aba6417d 100644 --- a/src/google/protobuf/feature_resolver.h +++ b/src/google/protobuf/feature_resolver.h @@ -79,16 +79,9 @@ class PROTOBUF_EXPORT FeatureResolver { FeatureSet defaults_; }; - -namespace internal { -// Gets the default feature set for a given edition. -absl::StatusOr PROTOBUF_EXPORT GetEditionFeatureSetDefaults( - Edition edition, const FeatureSetDefaults& defaults); -} // namespace internal } // namespace protobuf } // namespace google #include "google/protobuf/port_undef.inc" #endif // GOOGLE_PROTOBUF_FEATURE_RESOLVER_H__ - diff --git a/src/google/protobuf/internal_feature_helper.h b/src/google/protobuf/internal_feature_helper.h index f183778424842..08e1eecc708b2 100644 --- a/src/google/protobuf/internal_feature_helper.h +++ b/src/google/protobuf/internal_feature_helper.h @@ -26,6 +26,12 @@ class PROTOBUF_EXPORT InternalFeatureHelper { return desc.features(); } + template + static const FeatureSet& GetUnresolvedFeatures(const DescriptorT& desc) { + return desc.proto_features_ != nullptr ? *desc.proto_features_ + : FeatureSet::default_instance(); + } + private: friend class ::google::protobuf::compiler::CodeGenerator; friend class ::google::protobuf::compiler::CommandLineInterface; diff --git a/src/google/protobuf/json/internal/descriptor_traits.h b/src/google/protobuf/json/internal/descriptor_traits.h index 3a63b9bdbc59c..a2c5620a428c0 100644 --- a/src/google/protobuf/json/internal/descriptor_traits.h +++ b/src/google/protobuf/json/internal/descriptor_traits.h @@ -19,11 +19,13 @@ #include #include "google/protobuf/type.pb.h" +#include "google/protobuf/descriptor.pb.h" #include "absl/algorithm/container.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "google/protobuf/descriptor.h" @@ -411,14 +413,19 @@ struct Proto3Type { static const Desc& ContainingType(Field f) { return f->parent(); } static bool IsMap(Field f) { - if (f->proto().kind() != google::protobuf::Field::TYPE_MESSAGE) { + if (!IsRepeated(f) || + f->proto().kind() != google::protobuf::Field::TYPE_MESSAGE) { return false; } bool value = false; (void)WithFieldType(f, [&value](const Desc& desc) { - value = absl::c_any_of(desc.proto().options(), [&](auto& option) { - return option.name() == "map_entry"; + value = absl::c_any_of(desc.proto().options(), [&](const auto& option) { + // Per the docs this should only be "map_entry", but some code failed to + // heed this and included the full name. Support both. + return option.name() == "map_entry" || + option.name() == "google.protobuf.MessageOptions.map_entry" || + option.name() == "google.protobuf.MessageOptions.map_entry"; }); return absl::OkStatus(); }); @@ -431,6 +438,9 @@ struct Proto3Type { } static bool IsExplicitPresence(Field f) { + if (IsRepeated(f)) { + return false; + } // Implicit presence requires this weird check: in proto3 the following // cases support presence: // 1) Anything contained in a oneof (including things explicitly declared @@ -439,8 +449,13 @@ struct Proto3Type { // TYPE_MESSAGE here). if (f->parent().proto().syntax() == google::protobuf::SYNTAX_PROTO3) { return f->proto().oneof_index() != 0 || - (f->proto().kind() == google::protobuf::Field::TYPE_MESSAGE && - !IsRepeated(f)); + f->proto().kind() == google::protobuf::Field::TYPE_MESSAGE; + } + + if (f->parent().proto().syntax() == google::protobuf::SYNTAX_EDITIONS) { + return f->proto().kind() == google::protobuf::Field::TYPE_MESSAGE || + f->proto().oneof_index() != 0 || + f->features().field_presence() != google::protobuf::FeatureSet::IMPLICIT; } return f->proto().cardinality() == diff --git a/src/google/protobuf/json/internal/untyped_message.cc b/src/google/protobuf/json/internal/untyped_message.cc index a280976e1af99..6db6c641fafe9 100644 --- a/src/google/protobuf/json/internal/untyped_message.cc +++ b/src/google/protobuf/json/internal/untyped_message.cc @@ -18,14 +18,17 @@ #include #include "google/protobuf/type.pb.h" +#include "absl/algorithm/container.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" +#include "google/protobuf/descriptor.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/port.h" #include "google/protobuf/util/type_resolver.h" @@ -75,6 +78,18 @@ absl::Span ResolverPool::Message::FieldsByIndex() fields_[i].pool_ = pool_; fields_[i].raw_ = &raw_.fields(i); fields_[i].parent_ = this; + if (features_ != nullptr) { + fields_[i].features_ = std::make_unique(*features_); + auto status_or_features = GetFeatureSet( + fields_[i].raw_->options(), "google.protobuf.FieldOptions.features", + "google.protobuf.FieldOptions.features"); + if (status_or_features.ok()) { + fields_[i].features_->MergeFrom(*status_or_features); + } else { + // We cannot do much here. + status_or_features.IgnoreError(); + } + } } } @@ -137,6 +152,17 @@ absl::StatusOr ResolverPool::FindMessage( auto msg = absl::WrapUnique(new Message(this)); std::string url_buf(url); RETURN_IF_ERROR(resolver_->ResolveMessageType(url_buf, &msg->raw_)); + if (msg->raw_.syntax() == google::protobuf::SYNTAX_EDITIONS) { + ASSIGN_OR_RETURN(auto edition, ParseEdition(msg->raw_.edition())); + ASSIGN_OR_RETURN(const auto* default_feature_set, + GetDefaultFeatureSet(edition)); + msg->features_ = std::make_unique(*default_feature_set); + ASSIGN_OR_RETURN( + auto features, + GetFeatureSet(msg->raw_.options(), "google.protobuf.MessageOptions.features", + "google.protobuf.MessageOptions.features")); + msg->features_->MergeFrom(features); + } return messages_.try_emplace(std::move(url_buf), std::move(msg)) .first->second.get(); @@ -152,11 +178,79 @@ absl::StatusOr ResolverPool::FindEnum( auto enoom = absl::WrapUnique(new Enum(this)); std::string url_buf(url); RETURN_IF_ERROR(resolver_->ResolveEnumType(url_buf, &enoom->raw_)); + if (enoom->raw_.syntax() == google::protobuf::SYNTAX_EDITIONS) { + ASSIGN_OR_RETURN(auto edition, ParseEdition(enoom->raw_.edition())); + ASSIGN_OR_RETURN(const auto* default_feature_set, + GetDefaultFeatureSet(edition)); + enoom->features_ = std::make_unique(*default_feature_set); + ASSIGN_OR_RETURN( + auto features, + GetFeatureSet(enoom->raw_.options(), "google.protobuf.EnumOptions.features", + "google.protobuf.EnumOptions.features")); + enoom->features_->MergeFrom(features); + } return enums_.try_emplace(std::move(url_buf), std::move(enoom)) .first->second.get(); } +absl::StatusOr ResolverPool::ParseEdition( + absl::string_view edition_suffix) { + Edition edition; + if (!Edition_Parse(absl::StrCat("EDITION_", edition_suffix), &edition)) { + return absl::InvalidArgumentError( + absl::StrCat("unknown edition: %s", edition_suffix)); + } + return edition; +} + +absl::StatusOr ResolverPool::GetDefaultFeatureSet( + Edition edition) { + auto it = default_feature_sets_.find(edition); + if (it != default_feature_sets_.end()) { + return it->second.get(); + } + ASSIGN_OR_RETURN( + auto feature_set, + internal::GetEditionFeatureSetDefaults( + edition, google::protobuf::internal::cpp::GetFeatureSetDefaults())); + auto feature_set_ptr = + absl::WrapUnique(new FeatureSet(std::move(feature_set))); + return default_feature_sets_.try_emplace(edition, std::move(feature_set_ptr)) + .first->second.get(); +} + +absl::StatusOr ResolverPool::GetFeatureSet( + const RepeatedPtrField& options, + absl::string_view option_name, absl::string_view oss_option_name) { + auto option = absl::c_find_if( + options, [option_name, oss_option_name](const auto& option) { + return option.name() == "features" || option.name() == option_name || + option.name() == oss_option_name; + }); + if (option == options.end()) { + return FeatureSet(); + } + absl::string_view type_url = option->value().type_url(); + auto type_url_slash = type_url.find('/'); + if (type_url_slash == absl::string_view::npos || type_url_slash == 0) { + return absl::InvalidArgumentError(absl::StrCat( + "type_url must contain at least one / and a nonempty host; got: ", + type_url)); + } + absl::string_view type_name = type_url.substr(type_url_slash + 1); + if (type_name != "google.protobuf.FeatureSet") { + return absl::InvalidArgumentError(absl::StrCat( + "expected type name for option value to be google.protobuf.FeatureSet; got: ", + type_name)); + } + FeatureSet feature_set; + if (!feature_set.MergeFromString(option->value().value())) { + return absl::UnknownError("failed to merge feature set"); + } + return feature_set; +} + PROTOBUF_NOINLINE static absl::Status MakeEndGroupWithoutGroupError( int field_number) { return absl::InvalidArgumentError(absl::StrFormat( diff --git a/src/google/protobuf/json/internal/untyped_message.h b/src/google/protobuf/json/internal/untyped_message.h index 6a83029408033..4bc2e112c80d7 100644 --- a/src/google/protobuf/json/internal/untyped_message.h +++ b/src/google/protobuf/json/internal/untyped_message.h @@ -1,7 +1,9 @@ #ifndef GOOGLE_PROTOBUF_JSON_INTERNAL_UNTYPED_MESSAGE_H__ #define GOOGLE_PROTOBUF_JSON_INTERNAL_UNTYPED_MESSAGE_H__ +#include "google/protobuf/descriptor.pb.h" #include "absl/log/absl_check.h" +#include "absl/status/statusor.h" // Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. All rights reserved. // @@ -71,6 +73,9 @@ class ResolverPool { const Message& parent() const { return *parent_; } const google::protobuf::Field& proto() const { return *raw_; } + const FeatureSet& features() const { + return features_ != nullptr ? *features_ : FeatureSet::default_instance(); + } private: friend class ResolverPool; @@ -81,6 +86,7 @@ class ResolverPool { const google::protobuf::Field* raw_ = nullptr; const Message* parent_ = nullptr; mutable const void* type_ = nullptr; + std::unique_ptr features_; }; class Message { @@ -106,6 +112,7 @@ class ResolverPool { mutable absl::flat_hash_map fields_by_name_; mutable absl::flat_hash_map fields_by_number_; + std::unique_ptr features_; }; class Enum { @@ -125,6 +132,7 @@ class ResolverPool { google::protobuf::Enum raw_; mutable absl::flat_hash_map values_; + std::unique_ptr features_; }; explicit ResolverPool(google::protobuf::util::TypeResolver* resolver) @@ -137,8 +145,18 @@ class ResolverPool { absl::StatusOr FindEnum(absl::string_view url); private: + static absl::StatusOr ParseEdition(absl::string_view edition_suffix); + + absl::StatusOr GetDefaultFeatureSet(Edition edition); + + static absl::StatusOr GetFeatureSet( + const RepeatedPtrField& options, + absl::string_view option_name, absl::string_view oss_option_name); + absl::flat_hash_map> messages_; absl::flat_hash_map> enums_; + absl::flat_hash_map> + default_feature_sets_; google::protobuf::util::TypeResolver* resolver_; }; diff --git a/src/google/protobuf/util/type_resolver_util.cc b/src/google/protobuf/util/type_resolver_util.cc index 9572be05eccef..4419a1fc5825e 100644 --- a/src/google/protobuf/util/type_resolver_util.cc +++ b/src/google/protobuf/util/type_resolver_util.cc @@ -8,6 +8,7 @@ #include "google/protobuf/util/type_resolver_util.h" #include +#include #include #include "google/protobuf/any.pb.h" @@ -15,13 +16,17 @@ #include "google/protobuf/type.pb.h" #include "google/protobuf/wrappers.pb.h" #include "google/protobuf/descriptor.pb.h" +#include "absl/container/inlined_vector.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" +#include "absl/types/optional.h" +#include "google/protobuf/descriptor.h" #include "google/protobuf/io/strtod.h" +#include "google/protobuf/repeated_ptr_field.h" #include "google/protobuf/util/type_resolver.h" // clang-format off @@ -49,6 +54,72 @@ using google::protobuf::Type; using google::protobuf::UInt32Value; using google::protobuf::UInt64Value; +struct FeaturesField { + const FieldDescriptor* descriptor; + FeatureSet features; +}; + +absl::optional PartiallyResolveFeatures( + const Descriptor& descriptor) { + const FileDescriptor* file = descriptor.file(); + absl::InlinedVector features; + const Descriptor* parent = &descriptor; + while (parent != nullptr) { + const auto& descriptor_features = + internal::GetUnresolvedFeatureSet(*parent); + if (&descriptor_features != &FeatureSet::default_instance()) { + features.push_back(descriptor_features); + } + parent = parent->containing_type(); + } + const auto& descriptor_features = internal::GetUnresolvedFeatureSet(*file); + if (&descriptor_features != &FeatureSet::default_instance()) { + features.push_back(descriptor_features); + } + if (features.empty()) { + return absl::nullopt; + } + FeatureSet feature_set; + for (auto it = features.rbegin(); it != features.rend(); ++it) { + feature_set.MergeFrom(*it); + } + return feature_set; +} + +absl::optional PartiallyResolveFeatures( + const EnumDescriptor& descriptor) { + const FileDescriptor* file = descriptor.file(); + absl::InlinedVector features; + { + const auto& descriptor_features = + internal::GetUnresolvedFeatureSet(descriptor); + if (&descriptor_features != &FeatureSet::default_instance()) { + features.push_back(descriptor_features); + } + } + const Descriptor* parent = descriptor.containing_type(); + while (parent != nullptr) { + const auto& descriptor_features = + internal::GetUnresolvedFeatureSet(*parent); + if (&descriptor_features != &FeatureSet::default_instance()) { + features.push_back(descriptor_features); + } + parent = parent->containing_type(); + } + const auto& descriptor_features = internal::GetUnresolvedFeatureSet(*file); + if (&descriptor_features != &FeatureSet::default_instance()) { + features.push_back(descriptor_features); + } + if (features.empty()) { + return absl::nullopt; + } + FeatureSet feature_set; + for (auto it = features.rbegin(); it != features.rend(); ++it) { + feature_set.MergeFrom(*it); + } + return feature_set; +} + template static WrapperT WrapValue(T value) { WrapperT wrapper; @@ -134,11 +205,20 @@ void ConvertOptionField(const Reflection* reflection, const Message& options, // Implementation details for Convert*Options. void ConvertOptionsInternal(const Message& options, + const absl::optional& features_field, RepeatedPtrField