diff --git a/CHANGES.txt b/CHANGES.txt index c71788c7deb57..8e77f6abeb10b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,39 @@ +2021-03-04 version 3.15.5 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + Ruby + * Fixed quadratic memory use in array append (#8379) + + PHP + * Fixed quadratic memory use in array append (#8379) + + C++ + * Do not disable RTTI by default in the CMake build (#8377) + +2021-03-02 version 3.15.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + Ruby + * Fixed SEGV when users pass nil messages (#8363) + * Fixed quadratic memory usage when appending to arrays (#8364) + + C++ + * Create a CMake option to control whether or not RTTI is enabled (#8361) + + PHP + * read_property() handler is not supposed to return NULL (#8362) + +2021-02-25 version 3.15.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + Ruby + * Ruby <2.7 now uses WeakMap too, which prevents memory leaks. (#8341) + +2021-02-23 version 3.15.2 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + Ruby + * Fix for FieldDescriptor.get(msg) (#8330) + + C++ + * Fix PROTOBUF_CONSTINIT macro redefinition (#8323) + 2021-02-05 version 3.15.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) Ruby diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index 45baa36197632..e76911cc3fa35 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '3.15.1' + s.version = '3.15.5' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = '3-Clause BSD License' diff --git a/Protobuf.podspec b/Protobuf.podspec index 0d8206f07e72a..7864d2c232175 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.15.1' + s.version = '3.15.5' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = '3-Clause BSD License' diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 52661f522d997..5c3b6e45129a2 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -45,6 +45,7 @@ option(protobuf_BUILD_CONFORMANCE "Build conformance tests" OFF) option(protobuf_BUILD_EXAMPLES "Build examples" OFF) option(protobuf_BUILD_PROTOC_BINARIES "Build libprotoc and protoc compiler" ON) option(protobuf_BUILD_LIBPROTOC "Build libprotoc" OFF) +option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF) if (BUILD_SHARED_LIBS) set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON) else (BUILD_SHARED_LIBS) @@ -117,6 +118,10 @@ endif() add_definitions(-DGOOGLE_PROTOBUF_CMAKE_BUILD) +if (protobuf_DISABLE_RTTI) + add_definitions(-DGOOGLE_PROTOBUF_NO_RTTI=1) +endif() + find_package(Threads REQUIRED) if (CMAKE_USE_PTHREADS_INIT) add_definitions(-DHAVE_PTHREAD) diff --git a/configure.ac b/configure.ac index f6ae5add9bd5b..cb0fc63459e23 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.15.1],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.15.5],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index e1cebacb276c9..87acdfeefe9c3 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.15.1 + 3.15.5 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/master/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index f7201007c521f..f7cf3cfd17429 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -4,7 +4,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.15.1 + 3.15.5 7.2 Google Inc. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 46a76deec4efa..33aa28195fba7 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.15.1 + 3.15.5 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index 15590d53ecdab..ffcbfe38625e4 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.15.1 + 3.15.5 protobuf-java diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 61250a6f79247..6a011d77087f4 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.15.1 + 3.15.5 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index 80e8427dbd824..654f9a43297c2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.15.1 + 3.15.5 pom Protocol Buffers [Parent] diff --git a/java/util/pom.xml b/java/util/pom.xml index 0d6985dd31268..1c004eeaf5988 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.15.1 + 3.15.5 protobuf-java-util diff --git a/js/package.json b/js/package.json index 189881f7e805f..0d9266252e0b9 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "google-protobuf", - "version": "3.15.1", + "version": "3.15.5", "description": "Protocol Buffers for JavaScript", "main": "google-protobuf.js", "files": [ diff --git a/php/ext/google/protobuf/message.c b/php/ext/google/protobuf/message.c index 7f27670320dc0..0f1f4c964e6bd 100644 --- a/php/ext/google/protobuf/message.c +++ b/php/ext/google/protobuf/message.c @@ -334,7 +334,7 @@ static zval *Message_read_property(PROTO_VAL *obj, PROTO_STR *member, Message* intern = PROTO_VAL_P(obj); const upb_fielddef *f = get_field(intern, member); - if (!f) return NULL; + if (!f) return &EG(uninitialized_zval); Message_get(intern, f, rv); return rv; } diff --git a/php/ext/google/protobuf/package.xml b/php/ext/google/protobuf/package.xml index 31440b9012f76..4d322769f7aa4 100644 --- a/php/ext/google/protobuf/package.xml +++ b/php/ext/google/protobuf/package.xml @@ -10,11 +10,11 @@ protobuf-opensource@google.com yes - 2021-02-19 - + 2021-03-04 + - 3.15.1 - 3.15.1 + 3.15.5 + 3.15.5 stable @@ -22,15 +22,8 @@ 3-Clause BSD License - New changes in 3.15.0: - * unregister INI entries and fix invalid read on shutdown (#8042) - * Fix PhpDoc comments for message accessors to include "|null". (#8136) - * fix: convert native PHP floats to single precision (#8187) - * Fixed PHP to support field numbers >=2**28. (#8235) - * feat: add support for deprecated fields to PHP compiler (#8223) - * Protect against stack overflow if the user derives from Message. (#8248) - * Fixed clone for Message, RepeatedField, and MapField. (#8245) - * Updated upb to allow nonzero offset minutes in JSON timestamps. (#8258) + New changes in 3.15.5: + * Fixed quadratic memory use in array append (#8379) @@ -820,5 +813,65 @@ G A release. + + + 3.15.2 + 3.15.2 + + + stable + stable + + 2021-02-23 + + 3-Clause BSD License + + + + + + 3.15.3 + 3.15.3 + + + stable + stable + + 2021-02-24 + + 3-Clause BSD License + + + + + + 3.15.4 + 3.15.4 + + + stable + stable + + 2021-03-02 + + 3-Clause BSD License + + + + + + 3.15.5 + 3.15.5 + + + stable + stable + + 2021-03-04 + + 3-Clause BSD License + + + diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index c56a567ca38a3..913dfad7e97aa 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -7047,10 +7047,9 @@ void upb_array_set(upb_array *arr, size_t i, upb_msgval val) { } bool upb_array_append(upb_array *arr, upb_msgval val, upb_arena *arena) { - if (!_upb_array_realloc(arr, arr->len + 1, arena)) { + if (!upb_array_resize(arr, arr->len + 1, arena)) { return false; } - arr->len++; upb_array_set(arr, arr->len - 1, val); return true; } diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index de783ee08bebe..104a0d6ff7a20 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -76,7 +76,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.15.1" +#define PHP_PROTOBUF_VERSION "3.15.5" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index 4a5ed9314571f..ba9eaf8ac5843 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.15.1 + 3.15.5 pom Protobuf Compiler diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 22a6072ffd235..8c49968e59ffe 100644 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.15.1' +__version__ = '3.15.5' diff --git a/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb b/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb index b4a158f37ccd4..4f70f52dc4a96 100755 --- a/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb +++ b/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb @@ -326,18 +326,6 @@ def test_collect! end end - def test_compact! - m = TestMessage.new - m.repeated_msg << TestMessage2.new(:foo => 1) - m.repeated_msg << nil - m.repeated_msg << TestMessage2.new(:foo => 2) - reference_arr = m.repeated_string.to_a - - check_self_modifying_method(m.repeated_string, reference_arr) do |arr| - arr.compact! - end - end - def test_delete m = TestMessage.new reference_arr = %w(foo bar baz) diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index 6cf8174ccd56b..1d912c1a2e1ae 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -295,7 +295,7 @@ static VALUE DescriptorPool_alloc(VALUE klass) { self->def_to_descriptor = rb_hash_new(); self->symtab = upb_symtab_new(); - ObjectCache_Add(self->symtab, ret, _upb_symtab_arena(self->symtab)); + ObjectCache_Add(self->symtab, ret); return ret; } @@ -960,16 +960,14 @@ static VALUE FieldDescriptor_subtype(VALUE _self) { static VALUE FieldDescriptor_get(VALUE _self, VALUE msg_rb) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); const upb_msgdef *m; - const upb_msgdef *msg = Message_Get(msg_rb, &m); - VALUE arena = Message_GetArena(msg_rb); - upb_msgval msgval; + + Message_Get(msg_rb, &m); if (m != upb_fielddef_containingtype(self->fielddef)) { rb_raise(cTypeError, "get method called on wrong message type"); } - msgval = upb_msg_get(msg, self->fielddef); - return Convert_UpbToRuby(msgval, TypeInfo_get(self->fielddef), arena); + return Message_getfield(msg_rb, self->fielddef); } /* diff --git a/ruby/ext/google/protobuf_c/map.c b/ruby/ext/google/protobuf_c/map.c index 9d7d16b529233..9d0b37e10fb7f 100644 --- a/ruby/ext/google/protobuf_c/map.c +++ b/ruby/ext/google/protobuf_c/map.c @@ -93,7 +93,7 @@ VALUE Map_GetRubyWrapper(upb_map* map, upb_fieldtype_t key_type, if (val == Qnil) { val = Map_alloc(cMap); Map* self; - ObjectCache_Add(map, val, Arena_get(arena)); + ObjectCache_Add(map, val); TypedData_Get_Struct(val, Map, &Map_type, self); self->map = map; self->arena = arena; @@ -318,7 +318,7 @@ static VALUE Map_init(int argc, VALUE* argv, VALUE _self) { self->map = upb_map_new(Arena_get(self->arena), self->key_type, self->value_type_info.type); - ObjectCache_Add(self->map, _self, Arena_get(self->arena)); + ObjectCache_Add(self->map, _self); if (init_arg != Qnil) { Map_merge_into_self(_self, init_arg); @@ -590,9 +590,10 @@ VALUE Map_eq(VALUE _self, VALUE _other) { */ static VALUE Map_freeze(VALUE _self) { Map* self = ruby_to_Map(_self); - - ObjectCache_Pin(self->map, _self, Arena_get(self->arena)); - RB_OBJ_FREEZE(_self); + if (!RB_OBJ_FROZEN(_self)) { + Arena_Pin(self->arena, _self); + RB_OBJ_FREEZE(_self); + } return _self; } diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index 259f5e666d689..b01741b61cec3 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -105,7 +105,7 @@ void Message_InitPtr(VALUE self_, upb_msg *msg, VALUE arena) { Message* self = ruby_to_Message(self_); self->msg = msg; self->arena = arena; - ObjectCache_Add(msg, self_, Arena_get(arena)); + ObjectCache_Add(msg, self_); } VALUE Message_GetArena(VALUE msg_rb) { @@ -292,7 +292,7 @@ static void Message_setfield(upb_msg* msg, const upb_fielddef* f, VALUE val, upb_msg_set(msg, f, msgval, arena); } -static VALUE Message_getfield(VALUE _self, const upb_fielddef* f) { +VALUE Message_getfield(VALUE _self, const upb_fielddef* f) { Message* self = ruby_to_Message(_self); // This is a special-case: upb_msg_mutable() for map & array are logically // const (they will not change what is serialized) but physically @@ -855,8 +855,10 @@ static VALUE Message_to_h(VALUE _self) { */ static VALUE Message_freeze(VALUE _self) { Message* self = ruby_to_Message(_self); - ObjectCache_Pin(self->msg, _self, Arena_get(self->arena)); - RB_OBJ_FREEZE(_self); + if (!RB_OBJ_FROZEN(_self)) { + Arena_Pin(self->arena, _self); + RB_OBJ_FREEZE(_self); + } return _self; } @@ -1248,7 +1250,9 @@ upb_msg* Message_deep_copy(const upb_msg* msg, const upb_msgdef* m, const upb_msg* Message_GetUpbMessage(VALUE value, const upb_msgdef* m, const char* name, upb_arena* arena) { - if (value == Qnil) return NULL; + if (value == Qnil) { + rb_raise(cTypeError, "nil message not allowed here."); + } VALUE klass = CLASS_OF(value); VALUE desc_rb = rb_ivar_get(klass, descriptor_instancevar_interned); diff --git a/ruby/ext/google/protobuf_c/message.h b/ruby/ext/google/protobuf_c/message.h index 551f41f96d82f..2ec440c869cc7 100644 --- a/ruby/ext/google/protobuf_c/message.h +++ b/ruby/ext/google/protobuf_c/message.h @@ -63,6 +63,9 @@ const upb_msg* Message_GetUpbMessage(VALUE value, const upb_msgdef* m, // object will reference |arena| and ensure that it outlives this object. VALUE Message_GetRubyWrapper(upb_msg* msg, const upb_msgdef* m, VALUE arena); +// Gets the given field from this message. +VALUE Message_getfield(VALUE _self, const upb_fielddef* f); + // Implements #inspect for this message, printing the text to |b|. void Message_PrintMessage(StringBuilder* b, const upb_msg* msg, const upb_msgdef* m); diff --git a/ruby/ext/google/protobuf_c/protobuf.c b/ruby/ext/google/protobuf_c/protobuf.c index 737cd284b868d..c27f30aa2d558 100644 --- a/ruby/ext/google/protobuf_c/protobuf.c +++ b/ruby/ext/google/protobuf_c/protobuf.c @@ -167,30 +167,55 @@ void StringBuilder_PrintMsgval(StringBuilder* b, upb_msgval val, // Arena // ----------------------------------------------------------------------------- -void Arena_free(void* data) { upb_arena_free(data); } +typedef struct { + upb_arena *arena; + VALUE pinned_objs; +} Arena; + +static void Arena_mark(void *data) { + Arena *arena = data; + rb_gc_mark(arena->pinned_objs); +} + +static void Arena_free(void *data) { + Arena *arena = data; + upb_arena_free(arena->arena); +} static VALUE cArena; const rb_data_type_t Arena_type = { "Google::Protobuf::Internal::Arena", - { NULL, Arena_free, NULL }, + { Arena_mark, Arena_free, NULL }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY, }; static VALUE Arena_alloc(VALUE klass) { - upb_arena *arena = upb_arena_new(); + Arena *arena = ALLOC(Arena); + arena->arena = upb_arena_new(); + arena->pinned_objs = Qnil; return TypedData_Wrap_Struct(klass, &Arena_type, arena); } upb_arena *Arena_get(VALUE _arena) { - upb_arena *arena; - TypedData_Get_Struct(_arena, upb_arena, &Arena_type, arena); - return arena; + Arena *arena; + TypedData_Get_Struct(_arena, Arena, &Arena_type, arena); + return arena->arena; } VALUE Arena_new() { return Arena_alloc(cArena); } +void Arena_Pin(VALUE _arena, VALUE obj) { + Arena *arena; + TypedData_Get_Struct(_arena, Arena, &Arena_type, arena); + if (arena->pinned_objs == Qnil) { + arena->pinned_objs = rb_ary_new(); + } + rb_ary_push(arena->pinned_objs, obj); +} + void Arena_register(VALUE module) { VALUE internal = rb_define_module_under(module, "Internal"); VALUE klass = rb_define_class_under(internal, "Arena", rb_cObject); @@ -209,122 +234,79 @@ void Arena_register(VALUE module) { // different wrapper objects for the same C object, which saves memory and // preserves object identity. // -// We use Hash and/or WeakMap for the cache. WeakMap is faster overall -// (probably due to removal being integrated with GC) but doesn't work for Ruby -// <2.7 (see note below). We need Hash for Ruby <2.7 and for cases where we -// need to GC-root the object (notably when the object has been frozen). +// We use WeakMap for the cache. For Ruby <2.7 we also need a secondary Hash +// to store WeakMap keys because Ruby <2.7 WeakMap doesn't allow non-finalizable +// keys. #if RUBY_API_VERSION_CODE >= 20700 -#define USE_WEAK_MAP 1 +#define USE_SECONDARY_MAP 0 #else -#define USE_WEAK_MAP 0 +#define USE_SECONDARY_MAP 1 #endif -static VALUE ObjectCache_GetKey(const void* key) { - char buf[sizeof(key)]; - memcpy(&buf, &key, sizeof(key)); - intptr_t key_int = (intptr_t)key; - PBRUBY_ASSERT((key_int & 3) == 0); - return LL2NUM(key_int >> 2); -} +#if USE_SECONDARY_MAP -// Strong object cache, uses regular Hash and GC-roots objects. -// - For Ruby <2.7, used for all objects. -// - For Ruby >=2.7, used only for frozen objects, so we preserve the "frozen" -// bit (since this information is not preserved at the upb level). +// Maps Numeric -> Object. The object is then used as a key into the WeakMap. +// This is needed for Ruby <2.7 where a number cannot be a key to WeakMap. +// The object is used only for its identity; it does not contain any data. +VALUE secondary_map = Qnil; -VALUE strong_obj_cache = Qnil; - -static void StrongObjectCache_Init() { - rb_gc_register_address(&strong_obj_cache); - strong_obj_cache = rb_hash_new(); +static void SecondaryMap_Init() { + rb_gc_register_address(&secondary_map); + secondary_map = rb_hash_new(); } -static void StrongObjectCache_Remove(void* key) { - VALUE key_rb = ObjectCache_GetKey(key); - PBRUBY_ASSERT(rb_hash_lookup(strong_obj_cache, key_rb) != Qnil); - rb_hash_delete(strong_obj_cache, key_rb); +static VALUE SecondaryMap_Get(VALUE key) { + VALUE ret = rb_hash_lookup(secondary_map, key); + if (ret == Qnil) { + ret = rb_eval_string("Object.new"); + rb_hash_aset(secondary_map, key, ret); + } + return ret; } -static VALUE StrongObjectCache_Get(const void* key) { - VALUE key_rb = ObjectCache_GetKey(key); - return rb_hash_lookup(strong_obj_cache, key_rb); -} +#endif -static void StrongObjectCache_Add(const void* key, VALUE val, - upb_arena* arena) { - PBRUBY_ASSERT(StrongObjectCache_Get(key) == Qnil); - VALUE key_rb = ObjectCache_GetKey(key); - rb_hash_aset(strong_obj_cache, key_rb, val); - upb_arena_addcleanup(arena, (void*)key, StrongObjectCache_Remove); +static VALUE ObjectCache_GetKey(const void* key) { + char buf[sizeof(key)]; + memcpy(&buf, &key, sizeof(key)); + intptr_t key_int = (intptr_t)key; + PBRUBY_ASSERT((key_int & 3) == 0); + VALUE ret = LL2NUM(key_int >> 2); +#if USE_SECONDARY_MAP + ret = SecondaryMap_Get(ret); +#endif + return ret; } -// Weak object cache. This speeds up the test suite significantly, so we -// presume it speeds up real code also. However we can only use it in Ruby -// >=2.7 due to: -// https://bugs.ruby-lang.org/issues/16035 - -#if USE_WEAK_MAP +// Public ObjectCache API. VALUE weak_obj_cache = Qnil; +ID item_get; +ID item_set; -static void WeakObjectCache_Init() { +static void ObjectCache_Init() { rb_gc_register_address(&weak_obj_cache); VALUE klass = rb_eval_string("ObjectSpace::WeakMap"); weak_obj_cache = rb_class_new_instance(0, NULL, klass); -} - -static VALUE WeakObjectCache_Get(const void* key) { - VALUE key_rb = ObjectCache_GetKey(key); - VALUE ret = rb_funcall(weak_obj_cache, rb_intern("[]"), 1, key_rb); - return ret; -} - -static void WeakObjectCache_Add(const void* key, VALUE val) { - PBRUBY_ASSERT(WeakObjectCache_Get(key) == Qnil); - VALUE key_rb = ObjectCache_GetKey(key); - rb_funcall(weak_obj_cache, rb_intern("[]="), 2, key_rb, val); - PBRUBY_ASSERT(WeakObjectCache_Get(key) == val); -} - -#endif - -// Public ObjectCache API. - -static void ObjectCache_Init() { - StrongObjectCache_Init(); -#if USE_WEAK_MAP - WeakObjectCache_Init(); + item_get = rb_intern("[]"); + item_set = rb_intern("[]="); +#if USE_SECONDARY_MAP + SecondaryMap_Init(); #endif } -void ObjectCache_Add(const void* key, VALUE val, upb_arena *arena) { -#if USE_WEAK_MAP - (void)arena; - WeakObjectCache_Add(key, val); -#else - StrongObjectCache_Add(key, val, arena); -#endif +void ObjectCache_Add(const void* key, VALUE val) { + PBRUBY_ASSERT(ObjectCache_Get(key) == Qnil); + VALUE key_rb = ObjectCache_GetKey(key); + rb_funcall(weak_obj_cache, item_set, 2, key_rb, val); + PBRUBY_ASSERT(ObjectCache_Get(key) == val); } // Returns the cached object for this key, if any. Otherwise returns Qnil. VALUE ObjectCache_Get(const void* key) { -#if USE_WEAK_MAP - return WeakObjectCache_Get(key); -#else - return StrongObjectCache_Get(key); -#endif -} - -void ObjectCache_Pin(const void* key, VALUE val, upb_arena *arena) { -#if USE_WEAK_MAP - PBRUBY_ASSERT(WeakObjectCache_Get(key) == val); - // This will GC-root the object, but we'll still use the weak map for - // actual lookup. - StrongObjectCache_Add(key, val, arena); -#else - // Value is already pinned, nothing to do. -#endif + VALUE key_rb = ObjectCache_GetKey(key); + return rb_funcall(weak_obj_cache, item_get, 1, key_rb); } /* diff --git a/ruby/ext/google/protobuf_c/protobuf.h b/ruby/ext/google/protobuf_c/protobuf.h index 90fb0a2093353..e4873b34d2bda 100644 --- a/ruby/ext/google/protobuf_c/protobuf.h +++ b/ruby/ext/google/protobuf_c/protobuf.h @@ -55,6 +55,13 @@ const upb_fielddef* map_field_value(const upb_fielddef* field); VALUE Arena_new(); upb_arena *Arena_get(VALUE arena); +// Pins this Ruby object to the lifetime of this arena, so that as long as the +// arena is alive this object will not be collected. +// +// We use this to guarantee that the "frozen" bit on the object will be +// remembered, even if the user drops their reference to this precise object. +void Arena_Pin(VALUE arena, VALUE obj); + // ----------------------------------------------------------------------------- // ObjectCache // ----------------------------------------------------------------------------- @@ -68,19 +75,11 @@ upb_arena *Arena_get(VALUE arena); // Adds an entry to the cache. The "arena" parameter must give the arena that // "key" was allocated from. In Ruby <2.7.0, it will be used to remove the key // from the cache when the arena is destroyed. -void ObjectCache_Add(const void* key, VALUE val, upb_arena *arena); +void ObjectCache_Add(const void* key, VALUE val); // Returns the cached object for this key, if any. Otherwise returns Qnil. VALUE ObjectCache_Get(const void* key); -// Pins the previously added object so it is GC-rooted. This turns the -// reference to "val" from weak to strong. We use this to guarantee that the -// "frozen" bit on the object will be remembered, even if the user drops their -// reference to this precise object. -// -// The "arena" parameter must give the arena that "key" was allocated from. -void ObjectCache_Pin(const void* key, VALUE val, upb_arena *arena); - // ----------------------------------------------------------------------------- // StringBuilder, for inspect // ----------------------------------------------------------------------------- diff --git a/ruby/ext/google/protobuf_c/repeated_field.c b/ruby/ext/google/protobuf_c/repeated_field.c index 65ca3c664791d..da3e7ef0cdd7a 100644 --- a/ruby/ext/google/protobuf_c/repeated_field.c +++ b/ruby/ext/google/protobuf_c/repeated_field.c @@ -88,7 +88,7 @@ VALUE RepeatedField_GetRubyWrapper(upb_array* array, TypeInfo type_info, if (val == Qnil) { val = RepeatedField_alloc(cRepeatedField); RepeatedField* self; - ObjectCache_Add(array, val, Arena_get(arena)); + ObjectCache_Add(array, val); TypedData_Get_Struct(val, RepeatedField, &RepeatedField_type, self); self->array = array; self->arena = arena; @@ -500,9 +500,10 @@ VALUE RepeatedField_eq(VALUE _self, VALUE _other) { */ static VALUE RepeatedField_freeze(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); - - ObjectCache_Pin(self->array, _self, Arena_get(self->arena)); - RB_OBJ_FREEZE(_self); + if (!RB_OBJ_FROZEN(_self)) { + Arena_Pin(self->arena, _self); + RB_OBJ_FREEZE(_self); + } return _self; } @@ -610,7 +611,7 @@ VALUE RepeatedField_init(int argc, VALUE* argv, VALUE _self) { self->type_info = TypeInfo_FromClass(argc, argv, 0, &self->type_class, &ary); self->array = upb_array_new(arena, self->type_info.type); - ObjectCache_Add(self->array, _self, arena); + ObjectCache_Add(self->array, _self); if (ary != Qnil) { if (!RB_TYPE_P(ary, T_ARRAY)) { diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index a7aeda2df2532..61762fcd9904e 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -6663,10 +6663,9 @@ void upb_array_set(upb_array *arr, size_t i, upb_msgval val) { } bool upb_array_append(upb_array *arr, upb_msgval val, upb_arena *arena) { - if (!_upb_array_realloc(arr, arr->len + 1, arena)) { + if (!upb_array_resize(arr, arr->len + 1, arena)) { return false; } - arr->len++; upb_array_set(arr, arr->len - 1, val); return true; } diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index eb6f794de96c9..7149cf4b2dc3e 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.15.1" + s.version = "3.15.5" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index 107084e664361..8ddf72b81e4e2 100755 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -52,10 +52,15 @@ def test_issue_8311_crash outer = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Outer").msgclass - outer_proto = outer.new( + outer.new( inners: [] - ) - outer_proto['inners'].to_s + )['inners'].to_s + + assert_raise Google::Protobuf::TypeError do + outer.new( + inners: [nil] + ).to_s + end end def test_has_field diff --git a/ruby/tests/repeated_field_test.rb b/ruby/tests/repeated_field_test.rb index 6307447bc3f3a..1df6e1d4f1549 100755 --- a/ruby/tests/repeated_field_test.rb +++ b/ruby/tests/repeated_field_test.rb @@ -339,18 +339,6 @@ def test_collect! end end - def test_compact! - m = TestMessage.new - m.repeated_msg << TestMessage2.new(:foo => 1) - m.repeated_msg << nil - m.repeated_msg << TestMessage2.new(:foo => 2) - reference_arr = m.repeated_string.to_a - - check_self_modifying_method(m.repeated_string, reference_arr) do |arr| - arr.compact! - end - end - def test_delete m = TestMessage.new reference_arr = %w(foo bar baz) diff --git a/ruby/tests/well_known_types_test.rb b/ruby/tests/well_known_types_test.rb index 3eafe095ad0f5..ea042eb024417 100755 --- a/ruby/tests/well_known_types_test.rb +++ b/ruby/tests/well_known_types_test.rb @@ -193,4 +193,12 @@ def test_struct_nested_init assert_equal false, s['b'][:y] assert_equal false, s[:b]['y'] end + + def test_b8325 + value_field = Google::Protobuf::ListValue.descriptor.lookup("values") + proto = Google::Protobuf::ListValue.new( + values: [Google::Protobuf::Value.new(string_value: "Hello")] + ) + assert_equal '[]', value_field.get(proto).inspect + end end diff --git a/src/Makefile.am b/src/Makefile.am index c845814ebf7e7..a59971a94bdb9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else PTHREAD_DEF = endif -PROTOBUF_VERSION = 26:1:0 +PROTOBUF_VERSION = 26:5:0 if GCC # Turn on all warnings except for sign comparison (we ignore sign comparison diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 88a0163ae66fc..76406e6b058cd 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index b0958af0c2195..42634c6cb9dad 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index 9f71b12c5147f..026619f67e481 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index e49e9bd2ef29f..4bfe37e05a8c5 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 74b5a92611dbd..e87e7b6d1c895 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 8ac8481baf6db..9703b50028f53 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index e743efd6c433f..99fc7237e347a 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 6b6d34db02014..42a7a66c81181 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -335,7 +335,7 @@ // Shared google3/opensource definitions. ////////////////////////////////////// -#define PROTOBUF_VERSION 3015001 +#define PROTOBUF_VERSION 3015005 #define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3015000 #define PROTOBUF_MIN_PROTOC_VERSION 3015000 #define PROTOBUF_VERSION_SUFFIX "" @@ -575,17 +575,6 @@ #define PROTOBUF_CONSTINIT #endif -#if defined(__cpp_constinit) -#define PROTOBUF_CONSTINIT constinit -#elif defined(__has_cpp_attribute) -#if __has_cpp_attribute(clang::require_constant_initialization) -#define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] -#endif -#endif -#ifndef PROTOBUF_CONSTINIT -#define PROTOBUF_CONSTINIT -#endif - // Some globals with an empty non-trivial destructor are annotated with // no_destroy for performance reasons. It reduces the cost of these globals in // non-opt mode and under sanitizers. diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index a6a9757ac6f54..4a553e4d774ee 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 6f2d53f31e8d3..0758c9cb9204f 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index a58a2843ac453..7b380f21ffc42 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -82,7 +82,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3015001 +#define GOOGLE_PROTOBUF_VERSION 3015005 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index e6bf5ee48bdf6..325b33dcce064 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index 8d402d8d7fc22..4cff1f48dc6de 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index 8dbc3cb89450c..65831786f854d 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3015001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3015005 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc.