Skip to content

Commit

Permalink
simplify namespace to use proto package
Browse files Browse the repository at this point in the history
  • Loading branch information
@the-other-tim-brown
the-other-tim-brown committed Aug 30, 2022
1 parent fc344fa commit 55997e7
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 48 deletions.
55 changes: 9 additions & 46 deletions ...tilities/src/main/java/org/apache/hudi/utilities/sources/helpers/ProtoConversionUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import com.google.protobuf.BoolValue;
import com.google.protobuf.ByteString;
import com.google.protobuf.BytesValue;
import com.google.protobuf.DescriptorProtos;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DoubleValue;
import com.google.protobuf.FloatValue;
Expand All @@ -39,7 +38,6 @@
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.Utf8;

import java.io.File;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -135,20 +133,20 @@ public Schema getSchema(Class c, boolean flattenWrappedPrimitives) {
});
}

private Schema getEnumSchema(Descriptors.EnumDescriptor d) {
List<String> symbols = new ArrayList<>(d.getValues().size());
for (Descriptors.EnumValueDescriptor e : d.getValues()) {
symbols.add(e.getName());
private Schema getEnumSchema(Descriptors.EnumDescriptor enumDescriptor) {
List<String> symbols = new ArrayList<>(enumDescriptor.getValues().size());
for (Descriptors.EnumValueDescriptor valueDescriptor : enumDescriptor.getValues()) {
symbols.add(valueDescriptor.getName());
}
return Schema.createEnum(d.getName(), null, getNamespace(d.getFile(), d.getContainingType()), symbols);
return Schema.createEnum(enumDescriptor.getName(), null, getNamespace(enumDescriptor.getFullName()), symbols);
}

private Schema getMessageSchema(Descriptors.Descriptor descriptor, Map<Descriptors.Descriptor, Schema> seen, boolean flattenWrappedPrimitives) {
if (seen.containsKey(descriptor)) {
return seen.get(descriptor);
}
Schema result = Schema.createRecord(descriptor.getName(), null,
getNamespace(descriptor.getFile(), descriptor.getContainingType()), false);
getNamespace(descriptor.getFullName()), false);

seen.put(descriptor, result);

Expand Down Expand Up @@ -353,44 +351,9 @@ private Object getWrappedValue(Object value) {
return valueAsMessage.getField(valueAsMessage.getDescriptorForType().getFields().get(0));
}

private String getNamespace(Descriptors.FileDescriptor fd, Descriptors.Descriptor containing) {
DescriptorProtos.FileOptions filedOptions = fd.getOptions();
String classPackage = filedOptions.hasJavaPackage() ? filedOptions.getJavaPackage() : fd.getPackage();
String outer = "";
if (!filedOptions.getJavaMultipleFiles()) {
if (filedOptions.hasJavaOuterClassname()) {
outer = filedOptions.getJavaOuterClassname();
} else {
outer = new File(fd.getName()).getName();
outer = outer.substring(0, outer.lastIndexOf('.'));
outer = toCamelCase(outer);
}
}
StringBuilder inner = new StringBuilder();
while (containing != null) {
if (inner.length() == 0) {
inner.insert(0, containing.getName());
} else {
inner.insert(0, containing.getName() + "$");
}
containing = containing.getContainingType();
}
String d1 = (!outer.isEmpty() || inner.length() != 0 ? "." : "");
String d2 = (!outer.isEmpty() && inner.length() != 0 ? "$" : "");
return classPackage + d1 + outer + d2 + inner;
}

private String toCamelCase(String s) {
String[] parts = s.split("_");
StringBuilder camelCaseString = new StringBuilder(s.length());
for (String part : parts) {
camelCaseString.append(cap(part));
}
return camelCaseString.toString();
}

private String cap(String s) {
return s.substring(0, 1).toUpperCase() + s.substring(1).toLowerCase();
private String getNamespace(String descriptorFullName) {
int lastDotIndex = descriptorFullName.lastIndexOf('.');
return descriptorFullName.substring(0, lastDotIndex);
}
}
}
2 changes: 1 addition & 1 deletion hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_flattened.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"type":"record","name":"Sample","namespace":"org.apache.hudi.utilities.test.proto","fields":[{"name":"primitive_double","type":"double","default":0.0},{"name":"primitive_float","type":"float","default":0.0},{"name":"primitive_int","type":"int","default":0},{"name":"primitive_long","type":"long","default":0},{"name":"primitive_unsigned_int","type":"long","default":0},{"name":"primitive_unsigned_long","type":"long","default":0},{"name":"primitive_signed_int","type":"int","default":0},{"name":"primitive_signed_long","type":"long","default":0},{"name":"primitive_fixed_int","type":"int","default":0},{"name":"primitive_fixed_long","type":"long","default":0},{"name":"primitive_fixed_signed_int","type":"int","default":0},{"name":"primitive_fixed_signed_long","type":"long","default":0},{"name":"primitive_boolean","type":"boolean","default":false},{"name":"primitive_string","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"primitive_bytes","type":"bytes","default":""},{"name":"repeated_primitive","type":{"type":"array","items":"int"},"default":[]},{"name":"map_primitive","type":{"type":"array","items":{"type":"record","name":"MapPrimitiveEntry","namespace":"org.apache.hudi.utilities.test.proto.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":"int","default":0}]}},"default":[]},{"name":"nested_message","type":["null",{"type":"record","name":"Nested","fields":[{"name":"nested_int","type":"int","default":0}]}],"default":null},{"name":"repeated_message","type":{"type":"array","items":"Nested"},"default":[]},{"name":"map_message","type":{"type":"array","items":{"type":"record","name":"MapMessageEntry","namespace":"org.apache.hudi.utilities.test.proto.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":["null","org.apache.hudi.utilities.test.proto.Nested"],"default":null}]}},"default":[]},{"name":"wrapped_string","type":["null",{"type":"string","avro.java.string":"String"}],"default":null},{"name":"wrapped_int","type":["null","int"],"default":null},{"name":"wrapped_long","type":["null","long"],"default":null},{"name":"wrapped_unsigned_int","type":["null","long"],"default":null},{"name":"wrapped_unsigned_long","type":["null","long"],"default":null},{"name":"wrapped_double","type":["null","double"],"default":null},{"name":"wrapped_float","type":["null","float"],"default":null},{"name":"wrapped_boolean","type":["null","boolean"],"default":null},{"name":"wrapped_bytes","type":["null","bytes"],"default":null},{"name":"enum","type":{"type":"enum","name":"SampleEnum","symbols":["FIRST","SECOND"]},"default":"FIRST"},{"name":"timestamp","type":["null",{"type":"record","name":"Timestamp","namespace":"com.google.protobuf","fields":[{"name":"seconds","type":"long","default":0},{"name":"nanos","type":"int","default":0}]}],"default":null}]}
{"type":"record","name":"Sample","namespace":"test","fields":[{"name":"primitive_double","type":"double","default":0.0},{"name":"primitive_float","type":"float","default":0.0},{"name":"primitive_int","type":"int","default":0},{"name":"primitive_long","type":"long","default":0},{"name":"primitive_unsigned_int","type":"long","default":0},{"name":"primitive_unsigned_long","type":"long","default":0},{"name":"primitive_signed_int","type":"int","default":0},{"name":"primitive_signed_long","type":"long","default":0},{"name":"primitive_fixed_int","type":"int","default":0},{"name":"primitive_fixed_long","type":"long","default":0},{"name":"primitive_fixed_signed_int","type":"int","default":0},{"name":"primitive_fixed_signed_long","type":"long","default":0},{"name":"primitive_boolean","type":"boolean","default":false},{"name":"primitive_string","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"primitive_bytes","type":"bytes","default":""},{"name":"repeated_primitive","type":{"type":"array","items":"int"},"default":[]},{"name":"map_primitive","type":{"type":"array","items":{"type":"record","name":"MapPrimitiveEntry","namespace":"test.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":"int","default":0}]}},"default":[]},{"name":"nested_message","type":["null",{"type":"record","name":"Nested","fields":[{"name":"nested_int","type":"int","default":0}]}],"default":null},{"name":"repeated_message","type":{"type":"array","items":"Nested"},"default":[]},{"name":"map_message","type":{"type":"array","items":{"type":"record","name":"MapMessageEntry","namespace":"test.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":["null","test.Nested"],"default":null}]}},"default":[]},{"name":"wrapped_string","type":["null",{"type":"string","avro.java.string":"String"}],"default":null},{"name":"wrapped_int","type":["null","int"],"default":null},{"name":"wrapped_long","type":["null","long"],"default":null},{"name":"wrapped_unsigned_int","type":["null","long"],"default":null},{"name":"wrapped_unsigned_long","type":["null","long"],"default":null},{"name":"wrapped_double","type":["null","double"],"default":null},{"name":"wrapped_float","type":["null","float"],"default":null},{"name":"wrapped_boolean","type":["null","boolean"],"default":null},{"name":"wrapped_bytes","type":["null","bytes"],"default":null},{"name":"enum","type":{"type":"enum","name":"SampleEnum","symbols":["FIRST","SECOND"]},"default":"FIRST"},{"name":"timestamp","type":["null",{"type":"record","name":"Timestamp","namespace":"google.protobuf","fields":[{"name":"seconds","type":"long","default":0},{"name":"nanos","type":"int","default":0}]}],"default":null}]}
2 changes: 1 addition & 1 deletion hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_nested.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"type":"record","name":"Sample","namespace":"org.apache.hudi.utilities.test.proto","fields":[{"name":"primitive_double","type":"double","default":0.0},{"name":"primitive_float","type":"float","default":0.0},{"name":"primitive_int","type":"int","default":0},{"name":"primitive_long","type":"long","default":0},{"name":"primitive_unsigned_int","type":"long","default":0},{"name":"primitive_unsigned_long","type":"long","default":0},{"name":"primitive_signed_int","type":"int","default":0},{"name":"primitive_signed_long","type":"long","default":0},{"name":"primitive_fixed_int","type":"int","default":0},{"name":"primitive_fixed_long","type":"long","default":0},{"name":"primitive_fixed_signed_int","type":"int","default":0},{"name":"primitive_fixed_signed_long","type":"long","default":0},{"name":"primitive_boolean","type":"boolean","default":false},{"name":"primitive_string","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"primitive_bytes","type":"bytes","default":""},{"name":"repeated_primitive","type":{"type":"array","items":"int"},"default":[]},{"name":"map_primitive","type":{"type":"array","items":{"type":"record","name":"MapPrimitiveEntry","namespace":"org.apache.hudi.utilities.test.proto.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":"int","default":0}]}},"default":[]},{"name":"nested_message","type":["null",{"type":"record","name":"Nested","fields":[{"name":"nested_int","type":"int","default":0}]}],"default":null},{"name":"repeated_message","type":{"type":"array","items":"Nested"},"default":[]},{"name":"map_message","type":{"type":"array","items":{"type":"record","name":"MapMessageEntry","namespace":"org.apache.hudi.utilities.test.proto.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":["null","org.apache.hudi.utilities.test.proto.Nested"],"default":null}]}},"default":[]},{"name":"wrapped_string","type":["null",{"type":"record","name":"StringValue","namespace":"com.google.protobuf","fields":[{"name":"value","type":{"type":"string","avro.java.string":"String"},"default":""}]}],"default":null},{"name":"wrapped_int","type":["null",{"type":"record","name":"Int32Value","namespace":"com.google.protobuf","fields":[{"name":"value","type":"int","default":0}]}],"default":null},{"name":"wrapped_long","type":["null",{"type":"record","name":"Int64Value","namespace":"com.google.protobuf","fields":[{"name":"value","type":"long","default":0}]}],"default":null},{"name":"wrapped_unsigned_int","type":["null",{"type":"record","name":"UInt32Value","namespace":"com.google.protobuf","fields":[{"name":"value","type":"long","default":0}]}],"default":null},{"name":"wrapped_unsigned_long","type":["null",{"type":"record","name":"UInt64Value","namespace":"com.google.protobuf","fields":[{"name":"value","type":"long","default":0}]}],"default":null},{"name":"wrapped_double","type":["null",{"type":"record","name":"DoubleValue","namespace":"com.google.protobuf","fields":[{"name":"value","type":"double","default":0.0}]}],"default":null},{"name":"wrapped_float","type":["null",{"type":"record","name":"FloatValue","namespace":"com.google.protobuf","fields":[{"name":"value","type":"float","default":0.0}]}],"default":null},{"name":"wrapped_boolean","type":["null",{"type":"record","name":"BoolValue","namespace":"com.google.protobuf","fields":[{"name":"value","type":"boolean","default":false}]}],"default":null},{"name":"wrapped_bytes","type":["null",{"type":"record","name":"BytesValue","namespace":"com.google.protobuf","fields":[{"name":"value","type":"bytes","default":""}]}],"default":null},{"name":"enum","type":{"type":"enum","name":"SampleEnum","symbols":["FIRST","SECOND"]},"default":"FIRST"},{"name":"timestamp","type":["null",{"type":"record","name":"Timestamp","namespace":"com.google.protobuf","fields":[{"name":"seconds","type":"long","default":0},{"name":"nanos","type":"int","default":0}]}],"default":null}]}
{"type":"record","name":"Sample","namespace":"test","fields":[{"name":"primitive_double","type":"double","default":0.0},{"name":"primitive_float","type":"float","default":0.0},{"name":"primitive_int","type":"int","default":0},{"name":"primitive_long","type":"long","default":0},{"name":"primitive_unsigned_int","type":"long","default":0},{"name":"primitive_unsigned_long","type":"long","default":0},{"name":"primitive_signed_int","type":"int","default":0},{"name":"primitive_signed_long","type":"long","default":0},{"name":"primitive_fixed_int","type":"int","default":0},{"name":"primitive_fixed_long","type":"long","default":0},{"name":"primitive_fixed_signed_int","type":"int","default":0},{"name":"primitive_fixed_signed_long","type":"long","default":0},{"name":"primitive_boolean","type":"boolean","default":false},{"name":"primitive_string","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"primitive_bytes","type":"bytes","default":""},{"name":"repeated_primitive","type":{"type":"array","items":"int"},"default":[]},{"name":"map_primitive","type":{"type":"array","items":{"type":"record","name":"MapPrimitiveEntry","namespace":"test.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":"int","default":0}]}},"default":[]},{"name":"nested_message","type":["null",{"type":"record","name":"Nested","fields":[{"name":"nested_int","type":"int","default":0}]}],"default":null},{"name":"repeated_message","type":{"type":"array","items":"Nested"},"default":[]},{"name":"map_message","type":{"type":"array","items":{"type":"record","name":"MapMessageEntry","namespace":"test.Sample","fields":[{"name":"key","type":{"type":"string","avro.java.string":"String"},"default":""},{"name":"value","type":["null","test.Nested"],"default":null}]}},"default":[]},{"name":"wrapped_string","type":["null",{"type":"record","name":"StringValue","namespace":"google.protobuf","fields":[{"name":"value","type":{"type":"string","avro.java.string":"String"},"default":""}]}],"default":null},{"name":"wrapped_int","type":["null",{"type":"record","name":"Int32Value","namespace":"google.protobuf","fields":[{"name":"value","type":"int","default":0}]}],"default":null},{"name":"wrapped_long","type":["null",{"type":"record","name":"Int64Value","namespace":"google.protobuf","fields":[{"name":"value","type":"long","default":0}]}],"default":null},{"name":"wrapped_unsigned_int","type":["null",{"type":"record","name":"UInt32Value","namespace":"google.protobuf","fields":[{"name":"value","type":"long","default":0}]}],"default":null},{"name":"wrapped_unsigned_long","type":["null",{"type":"record","name":"UInt64Value","namespace":"google.protobuf","fields":[{"name":"value","type":"long","default":0}]}],"default":null},{"name":"wrapped_double","type":["null",{"type":"record","name":"DoubleValue","namespace":"google.protobuf","fields":[{"name":"value","type":"double","default":0.0}]}],"default":null},{"name":"wrapped_float","type":["null",{"type":"record","name":"FloatValue","namespace":"google.protobuf","fields":[{"name":"value","type":"float","default":0.0}]}],"default":null},{"name":"wrapped_boolean","type":["null",{"type":"record","name":"BoolValue","namespace":"google.protobuf","fields":[{"name":"value","type":"boolean","default":false}]}],"default":null},{"name":"wrapped_bytes","type":["null",{"type":"record","name":"BytesValue","namespace":"google.protobuf","fields":[{"name":"value","type":"bytes","default":""}]}],"default":null},{"name":"enum","type":{"type":"enum","name":"SampleEnum","symbols":["FIRST","SECOND"]},"default":"FIRST"},{"name":"timestamp","type":["null",{"type":"record","name":"Timestamp","namespace":"google.protobuf","fields":[{"name":"seconds","type":"long","default":0},{"name":"nanos","type":"int","default":0}]}],"default":null}]}

0 comments on commit 55997e7

Please sign in to comment.