Skip to content

Commit

Permalink
Add support for options in CRuby, JRuby and FFI (#14594) (#14739)
Browse files Browse the repository at this point in the history 
Rewrrte and extension of #12828, with additional work for JRuby. Partially fixes #1198 by adding support for custom options. Handling of extensions will be handled in a follow up.

Also includes these unrelated fixes:
* Removes code echo between `google/protobuf/repeated_field.rb` and `google/protobuf/ffi/repeated_field.rb` by `require`'ing the former in the latter.
* Adds missing calles to `testFrozen()` from methods of `RepeatedField` under JRuby that mutate.
* Various typos in comments.

Closes #14594

COPYBARA_INTEGRATE_REVIEW=#14594 from protocolbuffers:add-support-for-options-in-ruby 16cc9e3
PiperOrigin-RevId: 580848874
  • Loading branch information
@JasonLunn
JasonLunn authored Nov 13, 2023
1 parent a29f47d commit 2495d4f
Show file tree
Hide file tree
Showing 27 changed files with 584 additions and 183 deletions.
133 changes: 131 additions & 2 deletions ruby/ext/google/protobuf_c/defs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ static VALUE rb_str_maybe_null(const char* s) {
}
return rb_str_new2(s);
}

static ID options_instancevar_interned;
// -----------------------------------------------------------------------------
// DescriptorPool.
// -----------------------------------------------------------------------------
Expand Down Expand Up @@ -192,6 +192,7 @@ static void DescriptorPool_register(VALUE module) {

rb_gc_register_address(&generated_pool);
generated_pool = rb_class_new_instance(0, NULL, klass);
options_instancevar_interned = rb_intern("options");
}

// -----------------------------------------------------------------------------
Expand Down Expand Up @@ -226,6 +227,35 @@ static Descriptor* ruby_to_Descriptor(VALUE val) {
return ret;
}

// Decode and return a frozen instance of a Descriptor Option for the given pool
static VALUE decode_options(VALUE self, const char* option_type, int size,
const char* bytes, VALUE descriptor_pool) {
VALUE options_rb = rb_ivar_get(self, options_instancevar_interned);
if (options_rb != Qnil) {
return options_rb;
}

static const char* prefix = "google.protobuf.";
char fullname
[/*strlen(prefix)*/ 16 +
/*strln(longest option type supported e.g. "MessageOptions")*/ 14 +
/*null terminator*/ 1];

snprintf(fullname, sizeof(fullname), "%s%s", prefix, option_type);
const upb_MessageDef* msgdef = upb_DefPool_FindMessageByName(
ruby_to_DescriptorPool(descriptor_pool)->symtab, fullname);
if (!msgdef) {
rb_raise(rb_eRuntimeError, "Cannot find %s in DescriptorPool", option_type);
}

VALUE desc_rb = get_msgdef_obj(descriptor_pool, msgdef);
const Descriptor* desc = ruby_to_Descriptor(desc_rb);

options_rb = Message_decode_bytes(size, bytes, 0, desc->klass, true);
rb_ivar_set(self, options_instancevar_interned, options_rb);
return options_rb;
}

/*
* call-seq:
* Descriptor.new => descriptor
Expand Down Expand Up @@ -374,6 +404,26 @@ static VALUE Descriptor_msgclass(VALUE _self) {
return self->klass;
}

/*
* call-seq:
* Descriptor.options => options
*
* Returns the `MessageOptions` for this `Descriptor`.
*/
static VALUE Descriptor_options(VALUE _self) {
Descriptor* self = ruby_to_Descriptor(_self);
const google_protobuf_MessageOptions* opts =
upb_MessageDef_Options(self->msgdef);
upb_Arena* arena = upb_Arena_New();
size_t size;
char* serialized =
google_protobuf_MessageOptions_serialize(opts, arena, &size);
VALUE message_options = decode_options(_self, "MessageOptions", size,
serialized, self->descriptor_pool);
upb_Arena_Free(arena);
return message_options;
}

static void Descriptor_register(VALUE module) {
VALUE klass = rb_define_class_under(module, "Descriptor", rb_cObject);
rb_define_alloc_func(klass, Descriptor_alloc);
Expand All @@ -385,6 +435,7 @@ static void Descriptor_register(VALUE module) {
rb_define_method(klass, "msgclass", Descriptor_msgclass, 0);
rb_define_method(klass, "name", Descriptor_name, 0);
rb_define_method(klass, "file_descriptor", Descriptor_file_descriptor, 0);
rb_define_method(klass, "options", Descriptor_options, 0);
rb_include_module(klass, rb_mEnumerable);
rb_gc_register_address(&cDescriptor);
cDescriptor = klass;
Expand Down Expand Up @@ -484,12 +535,31 @@ static VALUE FileDescriptor_syntax(VALUE _self) {
}
}

/*
* call-seq:
* FileDescriptor.options => options
*
* Returns the `FileOptions` for this `FileDescriptor`.
*/
static VALUE FileDescriptor_options(VALUE _self) {
FileDescriptor* self = ruby_to_FileDescriptor(_self);
const google_protobuf_FileOptions* opts = upb_FileDef_Options(self->filedef);
upb_Arena* arena = upb_Arena_New();
size_t size;
char* serialized = google_protobuf_FileOptions_serialize(opts, arena, &size);
VALUE file_options = decode_options(_self, "FileOptions", size, serialized,
self->descriptor_pool);
upb_Arena_Free(arena);
return file_options;
}

static void FileDescriptor_register(VALUE module) {
VALUE klass = rb_define_class_under(module, "FileDescriptor", rb_cObject);
rb_define_alloc_func(klass, FileDescriptor_alloc);
rb_define_method(klass, "initialize", FileDescriptor_initialize, 3);
rb_define_method(klass, "name", FileDescriptor_name, 0);
rb_define_method(klass, "syntax", FileDescriptor_syntax, 0);
rb_define_method(klass, "options", FileDescriptor_options, 0);
rb_gc_register_address(&cFileDescriptor);
cFileDescriptor = klass;
}
Expand Down Expand Up @@ -540,7 +610,7 @@ static VALUE FieldDescriptor_alloc(VALUE klass) {

/*
* call-seq:
* EnumDescriptor.new(c_only_cookie, pool, ptr) => EnumDescriptor
* FieldDescriptor.new(c_only_cookie, pool, ptr) => FieldDescriptor
*
* Creates a descriptor wrapper object. May only be called from C.
*/
Expand Down Expand Up @@ -841,6 +911,25 @@ static VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value) {
return Qnil;
}

/*
* call-seq:
* FieldDescriptor.options => options
*
* Returns the `FieldOptions` for this `FieldDescriptor`.
*/
static VALUE FieldDescriptor_options(VALUE _self) {
FieldDescriptor* self = ruby_to_FieldDescriptor(_self);
const google_protobuf_FieldOptions* opts =
upb_FieldDef_Options(self->fielddef);
upb_Arena* arena = upb_Arena_New();
size_t size;
char* serialized = google_protobuf_FieldOptions_serialize(opts, arena, &size);
VALUE field_options = decode_options(_self, "FieldOptions", size, serialized,
self->descriptor_pool);
upb_Arena_Free(arena);
return field_options;
}

static void FieldDescriptor_register(VALUE module) {
VALUE klass = rb_define_class_under(module, "FieldDescriptor", rb_cObject);
rb_define_alloc_func(klass, FieldDescriptor_alloc);
Expand All @@ -857,6 +946,7 @@ static void FieldDescriptor_register(VALUE module) {
rb_define_method(klass, "clear", FieldDescriptor_clear, 1);
rb_define_method(klass, "get", FieldDescriptor_get, 1);
rb_define_method(klass, "set", FieldDescriptor_set, 2);
rb_define_method(klass, "options", FieldDescriptor_options, 0);
rb_gc_register_address(&cFieldDescriptor);
cFieldDescriptor = klass;
}
Expand Down Expand Up @@ -956,12 +1046,32 @@ static VALUE OneofDescriptor_each(VALUE _self) {
return Qnil;
}

/*
* call-seq:
* OneofDescriptor.options => options
*
* Returns the `OneofOptions` for this `OneofDescriptor`.
*/
static VALUE OneOfDescriptor_options(VALUE _self) {
OneofDescriptor* self = ruby_to_OneofDescriptor(_self);
const google_protobuf_OneofOptions* opts =
upb_OneofDef_Options(self->oneofdef);
upb_Arena* arena = upb_Arena_New();
size_t size;
char* serialized = google_protobuf_OneofOptions_serialize(opts, arena, &size);
VALUE oneof_options = decode_options(_self, "OneofOptions", size, serialized,
self->descriptor_pool);
upb_Arena_Free(arena);
return oneof_options;
}

static void OneofDescriptor_register(VALUE module) {
VALUE klass = rb_define_class_under(module, "OneofDescriptor", rb_cObject);
rb_define_alloc_func(klass, OneofDescriptor_alloc);
rb_define_method(klass, "initialize", OneofDescriptor_initialize, 3);
rb_define_method(klass, "name", OneofDescriptor_name, 0);
rb_define_method(klass, "each", OneofDescriptor_each, 0);
rb_define_method(klass, "options", OneOfDescriptor_options, 0);
rb_include_module(klass, rb_mEnumerable);
rb_gc_register_address(&cOneofDescriptor);
cOneofDescriptor = klass;
Expand Down Expand Up @@ -1131,6 +1241,24 @@ static VALUE EnumDescriptor_enummodule(VALUE _self) {
return self->module;
}

/*
* call-seq:
* EnumDescriptor.options => options
*
* Returns the `EnumOptions` for this `EnumDescriptor`.
*/
static VALUE EnumDescriptor_options(VALUE _self) {
EnumDescriptor* self = ruby_to_EnumDescriptor(_self);
const google_protobuf_EnumOptions* opts = upb_EnumDef_Options(self->enumdef);
upb_Arena* arena = upb_Arena_New();
size_t size;
char* serialized = google_protobuf_EnumOptions_serialize(opts, arena, &size);
VALUE enum_options = decode_options(_self, "EnumOptions", size, serialized,
self->descriptor_pool);
upb_Arena_Free(arena);
return enum_options;
}

static void EnumDescriptor_register(VALUE module) {
VALUE klass = rb_define_class_under(module, "EnumDescriptor", rb_cObject);
rb_define_alloc_func(klass, EnumDescriptor_alloc);
Expand All @@ -1141,6 +1269,7 @@ static void EnumDescriptor_register(VALUE module) {
rb_define_method(klass, "each", EnumDescriptor_each, 0);
rb_define_method(klass, "enummodule", EnumDescriptor_enummodule, 0);
rb_define_method(klass, "file_descriptor", EnumDescriptor_file_descriptor, 0);
rb_define_method(klass, "options", EnumDescriptor_options, 0);
rb_include_module(klass, rb_mEnumerable);
rb_gc_register_address(&cEnumDescriptor);
cEnumDescriptor = klass;
Expand Down
39 changes: 37 additions & 2 deletions ruby/ext/google/protobuf_c/glue.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,43 @@
upb_Arena* Arena_create() { return upb_Arena_Init(NULL, 0, &upb_alloc_global); }

google_protobuf_FileDescriptorProto* FileDescriptorProto_parse(
const char* serialized_file_proto, size_t length) {
upb_Arena* arena = Arena_create();
const char* serialized_file_proto, size_t length, upb_Arena* arena) {
return google_protobuf_FileDescriptorProto_parse(serialized_file_proto,
length, arena);
}

char* EnumDescriptor_serialized_options(const upb_EnumDef* enumdef,
size_t* size, upb_Arena* arena) {
const google_protobuf_EnumOptions* opts = upb_EnumDef_Options(enumdef);
char* serialized = google_protobuf_EnumOptions_serialize(opts, arena, size);
return serialized;
}

char* FileDescriptor_serialized_options(const upb_FileDef* filedef,
size_t* size, upb_Arena* arena) {
const google_protobuf_FileOptions* opts = upb_FileDef_Options(filedef);
char* serialized = google_protobuf_FileOptions_serialize(opts, arena, size);
return serialized;
}

char* Descriptor_serialized_options(const upb_MessageDef* msgdef, size_t* size,
upb_Arena* arena) {
const google_protobuf_MessageOptions* opts = upb_MessageDef_Options(msgdef);
char* serialized =
google_protobuf_MessageOptions_serialize(opts, arena, size);
return serialized;
}

char* OneOfDescriptor_serialized_options(const upb_OneofDef* oneofdef,
size_t* size, upb_Arena* arena) {
const google_protobuf_OneofOptions* opts = upb_OneofDef_Options(oneofdef);
char* serialized = google_protobuf_OneofOptions_serialize(opts, arena, size);
return serialized;
}

char* FieldDescriptor_serialized_options(const upb_FieldDef* fielddef,
size_t* size, upb_Arena* arena) {
const google_protobuf_FieldOptions* opts = upb_FieldDef_Options(fielddef);
char* serialized = google_protobuf_FieldOptions_serialize(opts, arena, size);
return serialized;
}
20 changes: 20 additions & 0 deletions ruby/ext/google/protobuf_c/map.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,26 @@ static VALUE Map_freeze(VALUE _self) {
return _self;
}

/*
* Deep freezes the map and values recursively.
* Internal use only.
*/
VALUE Map_internal_deep_freeze(VALUE _self) {
Map* self = ruby_to_Map(_self);
Map_freeze(_self);
if (self->value_type_info.type == kUpb_CType_Message) {
size_t iter = kUpb_Map_Begin;
upb_MessageValue key, val;

while (upb_Map_Next(self->map, &key, &val, &iter)) {
VALUE val_val =
Convert_UpbToRuby(val, self->value_type_info, self->arena);
Message_internal_deep_freeze(val_val);
}
}
return _self;
}

/*
* call-seq:
* Map.hash => hash_value
Expand Down
3 changes: 3 additions & 0 deletions ruby/ext/google/protobuf_c/map.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,7 @@ extern VALUE cMap;
// Call at startup to register all types in this module.
void Map_register(VALUE module);

// Recursively freeze map
VALUE Map_internal_deep_freeze(VALUE _self);

#endif // RUBY_PROTOBUF_MAP_H_
46 changes: 39 additions & 7 deletions ruby/ext/google/protobuf_c/message.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -859,6 +859,32 @@ static VALUE Message_freeze(VALUE _self) {
return _self;
}

/*
* Deep freezes the message object recursively.
* Internal use only.
*/
VALUE Message_internal_deep_freeze(VALUE _self) {
Message* self = ruby_to_Message(_self);
Message_freeze(_self);

int n = upb_MessageDef_FieldCount(self->msgdef);
for (int i = 0; i < n; i++) {
const upb_FieldDef* f = upb_MessageDef_Field(self->msgdef, i);
VALUE field = Message_getfield(_self, f);

if (field != Qnil) {
if (upb_FieldDef_IsMap(f)) {
Map_internal_deep_freeze(field);
} else if (upb_FieldDef_IsRepeated(f)) {
RepeatedField_internal_deep_freeze(field);
} else if (upb_FieldDef_IsSubMessage(f)) {
Message_internal_deep_freeze(field);
}
}
}
return _self;
}

/*
* call-seq:
* Message.[](index) => value
Expand Down Expand Up @@ -911,7 +937,7 @@ static VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value) {
* MessageClass.decode(data, options) => message
*
* Decodes the given data (as a string containing bytes in protocol buffers wire
* format) under the interpretration given by this message class's definition
* format) under the interpretation given by this message class's definition
* and returns a message object with the corresponding field values.
* @param options [Hash] options for the decoder
* recursion_limit: set to maximum decoding depth for message (default is 64)
Expand Down Expand Up @@ -942,18 +968,24 @@ static VALUE Message_decode(int argc, VALUE* argv, VALUE klass) {
rb_raise(rb_eArgError, "Expected string for binary protobuf data.");
}

return Message_decode_bytes(RSTRING_LEN(data), RSTRING_PTR(data), options,
klass, /*freeze*/ false);
}

VALUE Message_decode_bytes(int size, const char* bytes, int options,
VALUE klass, bool freeze) {
VALUE msg_rb = initialize_rb_class_with_no_args(klass);
Message* msg = ruby_to_Message(msg_rb);

upb_DecodeStatus status =
upb_Decode(RSTRING_PTR(data), RSTRING_LEN(data), (upb_Message*)msg->msg,
upb_MessageDef_MiniTable(msg->msgdef), NULL, options,
Arena_get(msg->arena));

upb_DecodeStatus status = upb_Decode(bytes, size, (upb_Message*)msg->msg,
upb_MessageDef_MiniTable(msg->msgdef),
NULL, options, Arena_get(msg->arena));
if (status != kUpb_DecodeStatus_Ok) {
rb_raise(cParseError, "Error occurred during parsing");
}

if (freeze) {
Message_internal_deep_freeze(msg_rb);
}
return msg_rb;
}

Expand Down
7 changes: 7 additions & 0 deletions ruby/ext/google/protobuf_c/message.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,13 @@ VALUE build_module_from_enumdesc(VALUE _enumdesc);
// module.
VALUE MessageOrEnum_GetDescriptor(VALUE klass);

// Decodes a Message from a byte sequence.
VALUE Message_decode_bytes(int size, const char* bytes, int options,
VALUE klass, bool freeze);

// Recursively freeze message
VALUE Message_internal_deep_freeze(VALUE _self);

// Call at startup to register all types in this module.
void Message_register(VALUE protobuf);

Expand Down
Loading

0 comments on commit 2495d4f

Please sign in to comment.