Skip to content

Commit

Permalink
fix: interpret Integer and Float values for TIMESTAMP as microseconds
Browse files Browse the repository at this point in the history
  • Loading branch information
@agrawal-siddharth
agrawal-siddharth committed Jun 29, 2023
1 parent 50ca432 commit 5e4b3d4
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 9 deletions.
8 changes: 4 additions & 4 deletions ...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ private void fillField(
if (val instanceof String) {
Double parsed = Doubles.tryParse((String) val);
if (parsed != null) {
protoMsg.setField(fieldDescriptor, parsed.longValue() * 10000000);
protoMsg.setField(fieldDescriptor, parsed.longValue());
return;
}
TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val);
Expand All @@ -435,7 +435,7 @@ private void fillField(
protoMsg.setField(fieldDescriptor, val);
return;
} else if (val instanceof Integer) {
protoMsg.setField(fieldDescriptor, Long.valueOf((Integer) val) * 10000000);
protoMsg.setField(fieldDescriptor, Long.valueOf((Integer) val));
return;
}
}
Expand Down Expand Up @@ -684,7 +684,7 @@ private void fillRepeatedField(
if (val instanceof String) {
Double parsed = Doubles.tryParse((String) val);
if (parsed != null) {
protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue() * 10000000);
protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue());
} else {
TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val);
protoMsg.addRepeatedField(
Expand All @@ -695,7 +695,7 @@ private void fillRepeatedField(
} else if (val instanceof Long) {
protoMsg.addRepeatedField(fieldDescriptor, val);
} else if (val instanceof Integer) {
protoMsg.addRepeatedField(fieldDescriptor, ((Integer) val) * 10000000);
protoMsg.addRepeatedField(fieldDescriptor, Long.valueOf((Integer) val));
} else {
throwWrongFieldType(fieldDescriptor, currentScope, index);
}
Expand Down
79 changes: 74 additions & 5 deletions ...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,12 @@ public class JsonToProtoMessageTest {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_timestamp")
.build();
private final TableFieldSchema TEST_TIMESTAMP_REPEATED =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.TIMESTAMP)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_timestamp_repeated")
.build();
private final TableFieldSchema TEST_TIME =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.TIME)
Expand Down Expand Up @@ -759,17 +765,17 @@ public void testTimestamp() throws Exception {
TestTimestamp.newBuilder()
.setTestString(10L)
.setTestStringTZ(1648493279010000L)
.setTestLong(0L)
.setTestInt(1534806950000000L)
.setTestFloat(1534680695000000000L)
.setTestLong(1687984085000000L)
.setTestInt(153480695L)
.setTestFloat(153468069500L)
.setTestOffset(1649135171000000L)
.setTestTimezone(1649174771000000L)
.setTestSaformat(1534680660000000L)
.build();
JSONObject json = new JSONObject();
json.put("test_string", "1970-01-01 00:00:00.000010");
json.put("test_string_T_Z", "2022-03-28T18:47:59.01Z");
json.put("test_long", 0L);
json.put("test_long", 1687984085000000L);
json.put("test_int", 153480695);
json.put("test_float", "1.534680695e11");
json.put("test_offset", "2022-04-05T09:06:11+04:00");
Expand All @@ -781,6 +787,69 @@ public void testTimestamp() throws Exception {
assertEquals(expectedProto, protoMsg);
}

@Test
public void testTimestampRepeated() throws Exception {
TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_string_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_string_T_Z_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_long_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_int_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_float_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_offset_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_timezone_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_saformat_repeated")
.build())
.build();
TestRepeatedTimestamp expectedProto =
TestRepeatedTimestamp.newBuilder()
.addTestStringRepeated(10L)
.addTestStringTZRepeated(1648493279010000L)
.addTestLongRepeated(1687984085000000L)
.addTestIntRepeated(153480695L)
.addTestFloatRepeated(153468069500L)
.addTestOffsetRepeated(1649135171000000L)
.addTestTimezoneRepeated(1649174771000000L)
.addTestSaformatRepeated(1534680660000000L)
.build();
JSONObject json = new JSONObject();
json.put("test_string_repeated", new JSONArray(new String[] {"1970-01-01 00:00:00.000010"}));
json.put("test_string_T_Z_repeated", new JSONArray(new String[] {"2022-03-28T18:47:59.01Z"}));
json.put("test_long_repeated", new JSONArray(new Long[] {1687984085000000L}));
json.put("test_int_repeated", new JSONArray(new Integer[] {153480695}));
json.put("test_float_repeated", new JSONArray(new String[] {"1.534680695e11"}));
json.put("test_offset_repeated", new JSONArray(new String[] {"2022-04-05T09:06:11+04:00"}));
json.put("test_timezone_repeated", new JSONArray(new String[] {"2022-04-05 09:06:11 PST"}));
json.put("test_saformat_repeated", new JSONArray(new String[] {"2018/08/19 12:11"}));
DynamicMessage protoMsg =
JsonToProtoMessage.INSTANCE.convertToProtoMessage(
TestRepeatedTimestamp.getDescriptor(), tableSchema, json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testDate() throws Exception {
TableSchema tableSchema =
Expand Down Expand Up @@ -962,7 +1031,7 @@ public void testStructComplex() throws Exception {
.setTestNumeric(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456")))
.setTestGeo("POINT(1,1)")
.setTestTimestamp(123456780000000L)
.setTestTimestamp(12345678L)
.setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1)))
.setTestTimeStr(89332507144L)
.addTestNumericRepeated(
Expand Down
11 changes: 11 additions & 0 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,17 @@ message TestTimestamp {
optional int64 test_saformat = 8;
}

message TestRepeatedTimestamp {
repeated int64 test_string_repeated = 1;
repeated int64 test_string_t_z_repeated = 2;
repeated int64 test_long_repeated = 3;
repeated int64 test_int_repeated = 4;
repeated int64 test_float_repeated = 5;
repeated int64 test_offset_repeated = 6;
repeated int64 test_timezone_repeated = 7;
repeated int64 test_saformat_repeated = 8;
}

message TestDate {
optional int32 test_string = 1;
optional int32 test_long = 2;
Expand Down

0 comments on commit 5e4b3d4

Please sign in to comment.